From 5974f719c7d59217efb990228cb4c0b4982977dc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:32:25 +0000 Subject: [PATCH 1/6] feat(ui): show catalog node detail from the control API and finish lineage labels The catalog node page, the lineage inspector and node links across Catalog, Checks, Changes and Lineage now load from the catalog listing, node status, node SQL and check reads. SQL is read only where the location grants view.sql, so viewers cause no refused, audited reads. Graph cards show the node name without its kind prefix. Sections the server does not report (run history, jobs, history, drift without built SQL, ad-hoc runs) are hidden instead of claiming there are none. --- ui/src/lib/api/http-catalog.test.ts | 45 +++++++++++++- ui/src/lib/api/http-catalog.ts | 59 ++++++++++++++++++- ui/src/lib/api/http-client.test.ts | 37 +++++++++++- ui/src/lib/api/http-client.ts | 28 ++++++++- ui/src/lib/api/types/build.ts | 9 ++- .../lib/components/lineage/asset-node.svelte | 4 +- ui/src/lib/lineage/catalog-graph.ts | 2 + ui/src/lib/lineage/scoped-lineage.svelte | 6 +- .../routes/catalog/[name]/history-tab.svelte | 5 +- ui/src/routes/catalog/[name]/node-view.svelte | 27 +++++---- .../routes/catalog/[name]/overview-tab.svelte | 32 +++++----- ui/src/routes/catalog/[name]/runs-tab.svelte | 2 +- ui/src/routes/lineage/+page.svelte | 32 ++++++---- 13 files changed, 233 insertions(+), 55 deletions(-) diff --git a/ui/src/lib/api/http-catalog.test.ts b/ui/src/lib/api/http-catalog.test.ts index 6c5452fd76..12323431a7 100644 --- a/ui/src/lib/api/http-catalog.test.ts +++ b/ui/src/lib/api/http-catalog.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import type { components } from '$control/schema.d.ts'; -import { toCatalogNodes } from './http-catalog'; +import { findNode, nodeDetailOf, toCatalogNodes } from './http-catalog'; +import { toCheck } from './http-checks'; type W = components['schemas']; @@ -53,4 +54,46 @@ describe('catalog node mapping', () => { expect([orders.description, orders.tags, orders.columnCount, orders.relation]).toEqual(['One row per order', ['marts'], 4, { database: null, schema: 'main', name: 'orders' }]); expect([orders.stale, orders.checks, orders.lastOutcome, orders.group, orders.owner]).toEqual([null, null, null, null, []]); }); + + it('finds a node by key, or by bare name preferring the model', () => { + const nodes = toCatalogNodes([node('source:orders'), node('source:customers'), node('model:orders', { materialization: 'view' })]); + expect([findNode(nodes, 'source:orders')?.key, findNode(nodes, 'orders')?.key, findNode(nodes, 'customers')?.key, findNode(nodes, 'order')]).toEqual(['source:orders', 'model:orders', 'source:customers', null]); + }); +}); + +function check(id: string, target: string, outcome: string): W['CheckResult'] { + return { event_id: `e-${id}`, binding_key: id, audit_name: id, audit_definition_name: 'not_null', attachment_kind: 'model', attached_target_name: target, severity: 'error', evaluation_mode: 'violations', run_scope_phase: 'final', occurred_at: '2026-11-07T06:00:00Z', run_id: 'r', invocation_id: 'i', outcome, reused: 'false' }; +} + +function status(definition: W['NodeStatus']['definition'], physical: W['NodeStatus']['physical']): W['NodeStatus'] { + const build = { definition_hash: 'd', event_id: 'e', observed_at: '2026-11-07T06:00:00Z', run_id: 'sqb-run-1', schema_fingerprint: 's', source_event_id: 'x', version_hash: 'v1' }; + return { node_key: 'model:orders', snapshot_id: 'snap', build, definition, physical, source_data: { status: 'unknown', reason: 'not tracked' } }; +} + +describe('catalog node detail', () => { + const nodes = toCatalogNodes([node('model:orders', { materialization: 'table' }), node('model:order_totals', { materialization: 'view', dependencies: ['model:orders'] }), node('model:orders_by_day', { materialization: 'view' })]); + const checks = [check('a', 'orders', 'pass'), check('b', 'model:orders', 'fail'), check('c', 'orders', 'warn'), check('d', 'orders_by_day', 'fail'), { ...check('e', 'orders', 'fail'), attachment_kind: 'source' }].map(toCheck); + const observed: W['NodeStatus']['physical'] = { status: 'observed', observation: { event_id: 'o', observed_at: '2026-11-07T06:00:00Z', record: { name: 'orders', relation_type: 'table', state_schema: 'main', columns: [{ name: 'order_id', data_type: 'INTEGER' }] } } }; + + it('keeps only its own checks, matched by key or by bare name of the same kind, and counts them by newest outcome', () => { + const d = nodeDetailOf(nodes, nodes[0], null, null, checks); + expect(d.checkResults.map((c) => c.id)).toEqual(['a', 'b', 'c']); + expect(d.checks).toEqual({ total: 3, passing: 1, warning: 1, failing: 1 }); + expect(d.downstream).toEqual(['model:order_totals']); + }); + + it('reports a changed definition as stale and maps the warehouse columns', () => { + const changed = nodeDetailOf(nodes, nodes[0], status({ status: 'changed', current_local_hash: 'b', recorded_local_hash: 'a' }, observed), null, []); + expect(changed.stale.map((r) => [r.reason, r.since])).toEqual([['changed', '2026-11-07T06:00:00Z']]); + expect(changed.columns).toEqual([{ name: 'order_id', type: 'INTEGER', description: '', audits: [] }]); + const matched = nodeDetailOf(nodes, nodes[0], status({ status: 'matched', local_hash: 'a' }, { status: 'unavailable', reason: 'no catalog' }), null, []); + expect([matched.stale, matched.columns]).toEqual([[], []]); + }); + + it('leaves refused SQL out and unreported history unknown rather than empty', () => { + const refused = nodeDetailOf(nodes, nodes[0], null, null, []); + expect([refused.sql, refused.runs, refused.jobs, refused.history]).toEqual([null, null, null, null]); + const withSql = nodeDetailOf(nodes, nodes[0], null, { node_key: 'model:orders', snapshot_id: 'snap', raw_sql: 'select {{ x }}', compiled_sql: null }, []); + expect(withSql.sql).toEqual({ raw: 'select {{ x }}', compiled: 'select {{ x }}' }); + }); }); diff --git a/ui/src/lib/api/http-catalog.ts b/ui/src/lib/api/http-catalog.ts index bba6472397..def4b6e40a 100644 --- a/ui/src/lib/api/http-catalog.ts +++ b/ui/src/lib/api/http-catalog.ts @@ -1,5 +1,5 @@ import type { components } from '$control/schema.d.ts'; -import type { CatalogNode } from './types/build'; +import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, StaleReason } from './types/build'; import type { NodeKind } from './types/operate'; type Wire = components['schemas']; @@ -48,3 +48,60 @@ export function toCatalogNodes(wire: Wire['CatalogNodeRecord'][]): CatalogNode[] const listed = new Set(nodes.map((node) => node.key)); return nodes.map((node) => ({ ...node, upstream: node.upstream.filter((key) => listed.has(key)) })); } + +/** Passing, warning and failing by each check's newest result; checks without one count only towards the total. */ +export function checkCountsOf(checks: Check[]): CheckCounts { + const outcome = (c: Check) => c.latest?.outcome; + return { + total: checks.length, + passing: checks.filter((c) => outcome(c) === 'pass').length, + warning: checks.filter((c) => outcome(c) === 'warn').length, + failing: checks.filter((c) => ['fail', 'error'].includes(outcome(c) ?? '')).length + }; +} + +/** A definition that changed since its last recorded build here; other staleness is not reported by the server. */ +function staleOf(status: Wire['NodeStatus'] | null): StaleReason[] { + if (status?.definition.status !== 'changed' || !status.build) return []; + return [{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: status.build.observed_at }]; +} + +/** A node key without its kind, e.g. `customer_summary` for `model:customer_summary`. */ +export const shortName = (key: string) => key.slice(key.indexOf(':') + 1); + +/** The listed node a key or bare name refers to, preferring a model; migrations name models without their kind. */ +export function findNode(nodes: CatalogNode[], ref: string): CatalogNode | null { + return nodes.find((n) => n.key === ref) ?? nodes.find((n) => n.key === `model:${ref}`) ?? nodes.find((n) => shortName(n.key) === ref) ?? null; +} + +/** + * A listed node with its status and SQL reads and its checks. The server keeps no + * per-node run history, row counts, cost, jobs or migrations, so those stay empty or null and the + * pages hide them. `sql` is null when the person may not see SQL. + */ +export function nodeDetailOf(nodes: CatalogNode[], node: CatalogNode, status: Wire['NodeStatus'] | null, sql: Wire['NodeSql'] | null, checks: Check[]): CatalogNodeDetail { + // Checks name their node without its kind, so the attachment tells a source from a model of the same name. + const own = checks.filter((c) => c.node === node.key || `${c.attachment}:${c.node}` === node.key); + const physical = status?.physical.status === 'observed' ? status.physical.observation.record : null; + return { + ...node, + stale: staleOf(status), + checks: checkCountsOf(own), + downstream: nodes.filter((n) => n.upstream.includes(node.key)).map((n) => n.key), + recentOutcomes: [], + lastRows: null, + rowsDelta: null, + lastDurationMs: null, + cost: null, + columns: (physical?.columns ?? []).map((c) => ({ name: c.name, type: c.data_type, description: '', audits: [] })), + sql: sql?.raw_sql ? { raw: sql.raw_sql, compiled: sql.compiled_sql ?? sql.raw_sql } : null, + code: null, + builds: [], + jobs: null, + runs: null, + checkResults: own, + warehouse: null, + microbatch: null, + history: null + }; +} diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index 8d05b8dc33..5c146edf0f 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { HttpChangeFeed, feedAllowed } from './http-client'; +import { HttpChangeFeed, createHttpClient, feedAllowed } from './http-client'; import type { ChangedSignal, ConnectionState } from './signals'; /** A stand-in `EventSource` the test opens, fails and feeds by hand. */ @@ -237,3 +237,38 @@ describe('feedAllowed', () => { await expect(feedAllowed('', ORDERS, () => {})).rejects.toThrow('network down'); }); }); + +describe('getCatalogNode', () => { + /** Answers the reads a node page makes; anything else, such as node status, is refused. */ + function serve(actions: string[]) { + const paths: string[] = []; + const answers: [string, unknown][] = [ + ['/bootstrap', { login_mode: 'password', access: { name: 'vic', principal_id: 'p', actions: [], locations: [{ location_id: '0191f0aa-orders', name: 'Orders', target: 'development', actions }] } }], + ['/nodes', [{ node_key: 'model:orders', kind: 'model', name: 'orders', materialization: 'table', tags: [], dependencies: [], snapshot_id: 's' }]], + ['/checks', []], + ['/sql', { node_key: 'model:orders', snapshot_id: 's', raw_sql: 'select 1', compiled_sql: 'select 1' }] + ]; + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + const path = new URL(request.url).pathname; + paths.push(path); + const answer = answers.find(([end]) => path.endsWith(end)); + return answer ? Response.json(answer[1]) : Response.json({ error: 'refused' }, { status: 403 }); + }) + ); + return paths; + } + + it.each([ + [['view', 'view.sql'], 'select 1'], + [['view'], null] + ])('with %j reads SQL only when the location allows it, so a viewer causes no refused read', async (actions, raw) => { + const paths = serve(actions); + const client = createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }); + await client.bootstrap(); + const detail = await client.getCatalogNode(ORDERS, 'orders'); + expect([detail?.key, detail?.sql?.raw ?? null]).toEqual(['model:orders', raw]); + expect(paths.filter((p) => p.endsWith('/sql'))).toHaveLength(raw ? 1 : 0); + }); +}); diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 03baee8ca5..ba3c0cf2b6 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -2,13 +2,13 @@ import createFetch from 'openapi-fetch'; import type { components, paths } from '$control/schema.d.ts'; import { ApiError, type ApiClient } from './client'; import { checkDetailOf, toCheck, toCheckEvidence, toCheckSql } from './http-checks'; -import { toCatalogNodes } from './http-catalog'; +import { findNode, nodeDetailOf, toCatalogNodes } from './http-catalog'; import { toSnapshotChanges } from './http-changes'; import { jobDetailOf, toJob, toTick, toTrigger, triggerDetailOf } from './http-jobs'; import { RUN_WINDOW, runDetailOf, runRowPage, runsMatching, runTabCountsOf, toRunNode, toRuns } from './http-runs'; import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals'; import type { AuthMode, Bootstrap, Capability, Location, Scope } from './types'; -import type { CatalogNode, Check, CheckDetail, CheckEvidence, CheckSql, SnapshotChange } from './types/build'; +import type { CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, SnapshotChange } from './types/build'; import type { Job, JobDetail, RerunMode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState } from './types/operate'; type Wire = components['schemas']; @@ -38,6 +38,7 @@ const SUPPORTED: ReadonlySet = new Set([ 'getCheckSql', 'getCheckEvidence', 'listCatalogNodes', + 'getCatalogNode', 'listSnapshotChanges' ]); @@ -186,6 +187,8 @@ export type HttpClientOptions = { */ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOptions): ApiClient { const http = createFetch({ baseUrl, credentials: 'same-origin' }); + // Locations whose bootstrap actions include view.sql; reading SQL without it is refused and audited. + let sqlLocations = new Set(); const implemented = { supports: (method: keyof ApiClient) => SUPPORTED.has(method), changes: new HttpChangeFeed(baseUrl, (scope) => feedAllowed(baseUrl, scope, onUnauthorized)), @@ -193,7 +196,9 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt const { data, error, response } = await http.GET('/api/control/bootstrap'); if (response.status === 401) onUnauthorized(); if (!data) throw new ApiError(errorText(error, response), response.status); - return toBootstrap(data); + const bootstrap = toBootstrap(data); + sqlLocations = new Set(bootstrap.locations.filter((l) => l.capabilities?.includes('view.sql')).map((l) => l.id)); + return bootstrap; }, listRunRows: async (scope: Scope, query: RunRowQuery = {}): Promise => runRowPage(await listedRuns(scope), scope.locationId, query), runTabCounts: async (scope: Scope, query: RunRowQuery = {}): Promise => runTabCountsOf(await listedRuns(scope), scope.locationId, query), @@ -256,6 +261,23 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt if (!snapshots.data) throw new ApiError(errorText(snapshots.error, snapshots.response), snapshots.response.status); return toSnapshotChanges(snapshots.data, runs); }, + async getCatalogNode(scope: Scope, ref: string): Promise { + const nodes = await implemented.listCatalogNodes(scope); + const node = findNode(nodes, ref); + if (!node) return null; + const { locationId, target } = parseLocationKey(scope.locationId); + const path = { location_id: locationId, target }; + const [status, sql, checks] = await Promise.all([ + http.GET('/api/control/locations/{location_id}/targets/{target}/node-status', { params: { path, query: { node_key: node.key } } }), + sqlLocations.has(scope.locationId) + ? http.GET('/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql', { params: { path: { ...path, node_key: node.key } } }) + : null, + checkResults(scope).catch(() => []) + ]); + if (status.response.status === 401 || sql?.response.status === 401) onUnauthorized(); + // Status and SQL have their own permissions; a refusal leaves that part out rather than failing the page. + return nodeDetailOf(nodes, node, status.data ?? null, sql?.data ?? null, checks.map(toCheck)); + }, listChecks: async (scope: Scope): Promise => (await checkResults(scope)).map(toCheck), getCheck: async (scope: Scope, id: string): Promise => checkDetailOf(await checkResults(scope, id)), async getCheckSql(scope: Scope, checkId: string): Promise { diff --git a/ui/src/lib/api/types/build.ts b/ui/src/lib/api/types/build.ts index 2eb5eb15b8..906f9e77df 100644 --- a/ui/src/lib/api/types/build.ts +++ b/ui/src/lib/api/types/build.ts @@ -177,12 +177,15 @@ export type CatalogNodeDetail = Omit & { code: { file: string; source: string } | null; /** As built in this target, newest first. */ builds: BuiltDefinition[]; - jobs: NodeJob[]; - runs: NodeRunEntry[]; + /** Null when the server does not report which jobs select the node. */ + jobs: NodeJob[] | null; + /** Null when the server does not report the node's run history. */ + runs: NodeRunEntry[] | null; checkResults: Check[]; warehouse: WarehouseObject | null; microbatch: MicrobatchCoverage | null; - history: NodeHistory; + /** Null when the server does not report renames and moves. */ + history: NodeHistory | null; }; // ---- column lineage ---- diff --git a/ui/src/lib/components/lineage/asset-node.svelte b/ui/src/lib/components/lineage/asset-node.svelte index 182c52609f..4954794647 100644 --- a/ui/src/lib/components/lineage/asset-node.svelte +++ b/ui/src/lib/components/lineage/asset-node.svelte @@ -22,6 +22,8 @@ type AssetData = { name: string; + /** Shown instead of the key when set; the key stays in the tooltip. */ + label?: string; kind: string; /** The last build outcome; staleness is shown separately. */ status: 'ok' | 'fail' | 'warn' | 'run' | 'src' | 'none'; @@ -98,7 +100,7 @@
- {d.name} + {d.label ?? d.name} {#if f.kind}{kindShort}{/if}
diff --git a/ui/src/lib/lineage/catalog-graph.ts b/ui/src/lib/lineage/catalog-graph.ts index aa96ee033f..ca9ae070bf 100644 --- a/ui/src/lib/lineage/catalog-graph.ts +++ b/ui/src/lib/lineage/catalog-graph.ts @@ -2,6 +2,7 @@ import type { Edge, Node } from '@xyflow/svelte'; import type { CatalogNode, StaleReason } from '$lib/api'; +import { shortName } from '$lib/api/http-catalog'; import { kindLine, nodeIconKey } from '$lib/build-area/labels'; export type CardStatus = 'ok' | 'fail' | 'warn' | 'src' | 'none'; @@ -35,6 +36,7 @@ export function checksBadge(node: CatalogNode): { checks?: string; checksColor?: export function cardData(node: CatalogNode): Record { return { name: node.key, + label: shortName(node.key), kind: kindLine(node), group: node.group, nodeKind: node.kind, diff --git a/ui/src/lib/lineage/scoped-lineage.svelte b/ui/src/lib/lineage/scoped-lineage.svelte index 9db90a9060..127c5ed0d4 100644 --- a/ui/src/lib/lineage/scoped-lineage.svelte +++ b/ui/src/lib/lineage/scoped-lineage.svelte @@ -181,12 +181,12 @@
--select {selectExpr} - + onclick={() => onExecute(effectiveSelection.map((id) => graphNodes.find((n) => n.id === id)?.name ?? id))}>▷ Execute {effectiveSelection.length > 1 ? `(${effectiveSelection.length})` : ''}{/if}
diff --git a/ui/src/routes/catalog/[name]/history-tab.svelte b/ui/src/routes/catalog/[name]/history-tab.svelte index 8cc25043b3..26a1301a63 100644 --- a/ui/src/routes/catalog/[name]/history-tab.svelte +++ b/ui/src/routes/catalog/[name]/history-tab.svelte @@ -1,5 +1,5 @@
diff --git a/ui/src/routes/catalog/[name]/node-view.svelte b/ui/src/routes/catalog/[name]/node-view.svelte index a2c0cec463..0e360812bd 100644 --- a/ui/src/routes/catalog/[name]/node-view.svelte +++ b/ui/src/routes/catalog/[name]/node-view.svelte @@ -29,6 +29,10 @@ const res = createResource((client, scope) => client.getCatalogNode(scope, name), ['runs', 'checks', 'snapshot', 'node_status', 'warehouse_sync']); const n = $derived(res.data ?? null); + // Checks and migrations link nodes by bare name; settle on the node key so the graph and links agree. + $effect(() => { + if (n && n.key !== name) goto(`/catalog/${n.key}${page.url.search}`, { replaceState: true }); + }); const catalog = createResource((client, scope) => client.listCatalogNodes(scope), ['snapshot', 'runs']); const colLineage = createResource((client, scope) => client.getColumnLineage(scope, name), ['snapshot']); @@ -38,13 +42,13 @@ if (n.columns.length) t.push(['columns', 'Columns']); if (n.sql) t.push(['sql', 'SQL']); else if (n.code) t.push(['sql', 'Code']); - if (n.sql) t.push(['drift', 'Drift']); + if (n.sql && n.builds.length) t.push(['drift', 'Drift']); t.push(['lineage', 'Lineage']); - if (n.columns.length) t.push(['collineage', 'Column lineage']); - if (n.kind !== 'source') t.push(['runs', 'Runs']); + if (n.columns.length && app.client.supports('getColumnLineage')) t.push(['collineage', 'Column lineage']); + if (n.kind !== 'source' && n.runs) t.push(['runs', 'Runs']); t.push(['checks', `Checks${n.checkResults.length ? ` ${n.checkResults.length}` : ''}`]); if (n.warehouse) t.push(['warehouse', 'Warehouse']); - t.push(['history', 'History']); + if (n.history) t.push(['history', 'History']); return t; }); // The URL is the single source of truth for the tab, so back/forward and reloads restore it. @@ -69,6 +73,7 @@ }; }); + const canRun = app.client.supports('launchAdhocRun'); let runOpen = $state(false); let runSelection = $state([]); function execute(selection: string[]) { @@ -80,12 +85,12 @@ - {#if n && n.kind !== 'source'} + {#if n && n.kind !== 'source' && canRun} execute([name])}>▷ Build {/if} - +{#if canRun}{/if}
@@ -98,7 +103,7 @@ {:else if n.lastOutcome} - {:else} + {:else if !n.lastBuild} {/if} @@ -109,7 +114,7 @@
{#if n}
- {kindLine(n)}{n.group}{#if n.owner.length}{n.owner.join(', ')}{/if} + {kindLine(n)}{#if n.group}{n.group}{/if}{#if n.owner.length}{n.owner.join(', ')}{/if} {#if n.kind === 'source'}{n.sourceFreshness?.loadedAt ? `Loaded ${formatAgo(n.sourceFreshness.loadedAt, app.now)}` : 'No freshness data'} {:else}{n.lastBuild ? `Last built ${formatAgo(n.lastBuild.builtAt, app.now)}` : 'Never built'}{/if}
@@ -153,7 +158,7 @@ {:else if tab === 'lineage'} {#if graph.nodes.length} - + {/if} {:else if tab === 'collineage'} {#if colLineage.data} @@ -165,8 +170,8 @@ {:else if tab === 'warehouse' && n.warehouse} - {:else if tab === 'history'} - + {:else if tab === 'history' && n.history} + {/if}
{/if} diff --git a/ui/src/routes/catalog/[name]/overview-tab.svelte b/ui/src/routes/catalog/[name]/overview-tab.svelte index 270ad9db38..abf7e75a32 100644 --- a/ui/src/routes/catalog/[name]/overview-tab.svelte +++ b/ui/src/routes/catalog/[name]/overview-tab.svelte @@ -18,6 +18,8 @@ const source = $derived(node.kind === 'source'); const unique = $derived(node.sql?.raw.match(/unique_key\s+([^,\n]+)/)?.[1] ?? null); const freshnessTone = $derived(!node.sourceFreshness ? 'var(--sb-text-faint)' : node.sourceFreshness.status === 'pass' ? 'var(--sb-success)' : node.sourceFreshness.status === 'warn' ? 'var(--sb-warning)' : 'var(--sb-error)'); + // Drift compares the current SQL with the SQL of the last build, so it needs both. + const hasDrift = $derived(!!node.sql && node.builds.length > 0); const minutes = (s: number | null) => (s === null ? '—' : s >= 3600 && s % 3600 === 0 ? `${s / 3600}h` : `${Math.round(s / 60)}m`); @@ -41,24 +43,24 @@ {:else}
- {#if node.lastOutcome}{:else}{/if} + {#if node.lastOutcome}{:else if !node.lastBuild}{/if} Last built:
Rows{formatNumber(node.lastRows)}{#if node.rowsDelta} 0}>{node.rowsDelta > 0 ? '+' : ''}{formatNumber(node.rowsDelta)}{/if}
Duration{formatMs(node.lastDurationMs)}
Checks{node.checks.total ? `${node.checks.passing}/${node.checks.total}` : '—'}
-
Builds · 40{node.recentOutcomes.filter((o) => o === 'failed').length} failed
+ {#if node.recentOutcomes.length}
Builds · {node.recentOutcomes.length}{node.recentOutcomes.filter((o) => o === 'failed').length} failed
{/if} {#if node.cost}
Cost per build{formatCredits(node.cost.perBuild)}{formatCredits(node.cost.last24h)} / 24h
{/if}
{#if node.recentOutcomes.length}
{/if} {#each node.stale as r, i (i)} - {/each}
@@ -122,7 +124,7 @@
Kind
{kindLine(node)}
{#if unique}
Unique key
{unique}
{/if} -
Group
{node.group}
+ {#if node.group}
Group
{node.group}
{/if}
Owner
{node.owner.join(', ') || '—'}
Path
{node.path}
Relation
{relationName(node.relation)}
@@ -136,14 +138,16 @@
{#each node.tags as t (t)}{t}{/each}
{/if} -
-

Jobs

- {#each node.jobs as j (j.key)} - {j.key}{j.kind === 'group' ? 'group' : 'job'}{j.triggerIds.length ? ` · ${j.triggerIds.length} ${j.triggerIds.length === 1 ? 'trigger' : 'triggers'}` : ' · manual'} - {:else} -

No job selects this node.

- {/each} -
+ {#if node.jobs} +
+

Jobs

+ {#each node.jobs as j (j.key)} + {j.key}{j.kind === 'group' ? 'group' : 'job'}{j.triggerIds.length ? ` · ${j.triggerIds.length} ${j.triggerIds.length === 1 ? 'trigger' : 'triggers'}` : ' · manual'} + {:else} +

No job selects this node.

+ {/each} +
+ {/if} @@ -188,7 +192,7 @@ .job-row span { color: var(--primary); } .job-row small { margin-left: auto; color: var(--muted-foreground); font-size: 12px; } .drift-hint { display: flex; width: 100%; align-items: center; gap: 9px; margin-top: 12px; padding: 8px 10px; border: 1px solid color-mix(in srgb, var(--sb-stale) 35%, var(--border)); border-radius: 2px; background: transparent; color: var(--foreground); font-size: 12.5px; text-align: left; } - .drift-hint:hover { background: var(--sb-hover); } + .drift-hint:not(:disabled):hover { background: var(--sb-hover); } .drift-hint :global(svg) { flex: 0 0 auto; color: var(--sb-stale); } .drift-hint.changed_upstream { border-style: dashed; } .drift-hint.behind { border-color: color-mix(in srgb, var(--sb-warning) 35%, var(--border)); } diff --git a/ui/src/routes/catalog/[name]/runs-tab.svelte b/ui/src/routes/catalog/[name]/runs-tab.svelte index c736199e6b..23005d37ab 100644 --- a/ui/src/routes/catalog/[name]/runs-tab.svelte +++ b/ui/src/routes/catalog/[name]/runs-tab.svelte @@ -15,7 +15,7 @@ - {#each node.runs as r, i (`${r.runId ?? r.fingerprintRunId}-${i}`)} + {#each node.runs ?? [] as r, i (`${r.runId ?? r.fingerprintRunId}-${i}`)}
OutcomeRunJobRowsDurationStarted
diff --git a/ui/src/routes/lineage/+page.svelte b/ui/src/routes/lineage/+page.svelte index 3bb38719e1..18702af834 100644 --- a/ui/src/routes/lineage/+page.svelte +++ b/ui/src/routes/lineage/+page.svelte @@ -63,7 +63,7 @@ untrack(() => { if (!nodes.length) { const graph = catalogGraph(data); - const focus = data.some((n) => n.key === initialFocus) ? initialFocus : 'fact_orders'; + const focus = data.some((n) => n.key === initialFocus) ? initialFocus : null; nodes = graph.nodes.map((n) => (n.id === focus ? { ...n, selected: true } : n)); edges = graph.edges; return; @@ -83,7 +83,7 @@ const scaleModels = Number(page.url.searchParams.get('models') ?? 0); const scaleMode = scaleModels > 0; const FAR_ZOOM = 0.5; - let scaleStartId = 'fact_orders'; + let scaleStartId = ''; const scaleLayout = scaleMode ? (() => { const synthetic = scaleGraph(scaleModels); const t = performance.now(); @@ -253,6 +253,8 @@ const listed = $derived(detailSupported ? null : (catalogByKey.get(String(primarySlug)) ?? null)); const isSource = $derived(d?.kind === 'source'); const hasSql = $derived(!!d?.sql); + // Drift compares the current SQL with the SQL of the last build, so it needs both. + const hasDrift = $derived(hasSql && !!d?.builds.length); // ---- toolbar filters: non-matching nodes are dimmed ---- let kindFilter = $state(''); @@ -484,7 +486,7 @@ {#each browserNodes.slice(0, 8) as node (node.id)} {:else} @@ -496,8 +498,8 @@ {#if !scaleMode}
- - + {#if groupOptions.length}{/if} + {#if ownerOptions.length}{/if} ({ ...o, count: catalogNodes.filter((c) => matchesFilterStatus(c, o.value)).length }))} />
{/if} @@ -550,7 +552,7 @@ title={String(node.data.name)} > - {node.data.name} + {node.data.label ?? node.data.name} {/each} {/if} @@ -621,16 +623,19 @@ {:else}
- {primary?.data.name} + {primary?.data.label ?? primary?.data.name} {#if d}Catalog ↗{/if}
{d ? kindLine(d) : primary?.data.kind} - {#if d}{d.group}{#if d.owner.length}{d.owner.join(', ')}{/if}{/if} + {#if d}{#if d.group}{d.group}{/if}{#if d.owner.length}{d.owner.join(', ')}{/if}{/if}
{/if}
+ {#if !scaleMode && !primary} +

Select a node to see its details.

 + {:else} {#if scaleMode}

A synthetic graph for layout and rendering at scale; nodes have no catalog detail.

@@ -660,7 +665,7 @@ {#snippet meta()}{#if d.lastBuild?.runId}View run →{/if}{/snippet}
- {#if d.lastOutcome}{:else}{/if} + {#if d.lastOutcome}{:else if !d.lastBuild}{/if}
@@ -673,10 +678,10 @@
{/if} {#each d.stale as r, i (i)} - {/each}
@@ -728,7 +733,7 @@ {/if} - {#if d && !isSource} + {#if d && !isSource && d.runs} {#snippet meta()}All →{/snippet} {#each d.runs.slice(0, 3) as run, i (i)} @@ -767,13 +772,14 @@
Kind
{kindLine(d)}
Unique key
{sqlOutline.config.find((c) => c.key === 'unique_key')?.value ?? '—'}
-
Group
{d.group}
+
Group
{d.group ?? '—'}
Owner
{d.owner.join(', ') || '—'}
Tags
{d.tags.join(', ') || '—'}
Build version
{d.lastBuild?.fingerprint.slice(0, 12) ?? '—'}
 {/if} + {/if}
From f729c4c064767ff9584cae578d49ce1ecee7cb02 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:37:28 +0000 Subject: [PATCH 2/6] fix(ui): ask for refused node SQL once until reload view.sql in a location's bootstrap actions may hold on only some nodes, so the node SQL read can still be refused. Each refusal is audited and node pages refetch on change signals, so remember the refused node and skip its SQL read until the page reloads. --- ui/src/lib/api/http-client.test.ts | 14 ++++++++++++-- ui/src/lib/api/http-client.ts | 7 +++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index 5c146edf0f..2105944003 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -240,7 +240,7 @@ describe('feedAllowed', () => { describe('getCatalogNode', () => { /** Answers the reads a node page makes; anything else, such as node status, is refused. */ - function serve(actions: string[]) { + function serve(actions: string[], sqlStatus = 200) { const paths: string[] = []; const answers: [string, unknown][] = [ ['/bootstrap', { login_mode: 'password', access: { name: 'vic', principal_id: 'p', actions: [], locations: [{ location_id: '0191f0aa-orders', name: 'Orders', target: 'development', actions }] } }], @@ -254,7 +254,8 @@ describe('getCatalogNode', () => { const path = new URL(request.url).pathname; paths.push(path); const answer = answers.find(([end]) => path.endsWith(end)); - return answer ? Response.json(answer[1]) : Response.json({ error: 'refused' }, { status: 403 }); + const refused = !answer || (answer[0] === '/sql' && sqlStatus === 403); + return refused ? Response.json({ error: 'refused' }, { status: 403 }) : Response.json(answer[1]); }) ); return paths; @@ -271,4 +272,13 @@ describe('getCatalogNode', () => { expect([detail?.key, detail?.sql?.raw ?? null]).toEqual(['model:orders', raw]); expect(paths.filter((p) => p.endsWith('/sql'))).toHaveLength(raw ? 1 : 0); }); + + it('asks for a refused node SQL once, when view.sql holds on only some nodes', async () => { + const paths = serve(['view', 'view.sql'], 403); + const client = createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }); + await client.bootstrap(); + const loads = [await client.getCatalogNode(ORDERS, 'orders'), await client.getCatalogNode(ORDERS, 'orders')]; + expect(loads.map((d) => [d?.key, d?.sql])).toEqual([['model:orders', null], ['model:orders', null]]); + expect(paths.filter((p) => p.endsWith('/sql'))).toHaveLength(1); + }); }); diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index ba3c0cf2b6..952caa2133 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -187,8 +187,10 @@ export type HttpClientOptions = { */ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOptions): ApiClient { const http = createFetch({ baseUrl, credentials: 'same-origin' }); - // Locations whose bootstrap actions include view.sql; reading SQL without it is refused and audited. + // Locations whose bootstrap actions include view.sql, which may hold on only some nodes, and the + // nodes there whose SQL was refused; each refused read is audited, so neither is asked again until reload. let sqlLocations = new Set(); + const sqlRefused = new Set(); const implemented = { supports: (method: keyof ApiClient) => SUPPORTED.has(method), changes: new HttpChangeFeed(baseUrl, (scope) => feedAllowed(baseUrl, scope, onUnauthorized)), @@ -269,12 +271,13 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt const path = { location_id: locationId, target }; const [status, sql, checks] = await Promise.all([ http.GET('/api/control/locations/{location_id}/targets/{target}/node-status', { params: { path, query: { node_key: node.key } } }), - sqlLocations.has(scope.locationId) + sqlLocations.has(scope.locationId) && !sqlRefused.has(`${scope.locationId}|${node.key}`) ? http.GET('/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql', { params: { path: { ...path, node_key: node.key } } }) : null, checkResults(scope).catch(() => []) ]); if (status.response.status === 401 || sql?.response.status === 401) onUnauthorized(); + if (sql?.response.status === 403) sqlRefused.add(`${scope.locationId}|${node.key}`); // Status and SQL have their own permissions; a refusal leaves that part out rather than failing the page. return nodeDetailOf(nodes, node, status.data ?? null, sql?.data ?? null, checks.map(toCheck)); }, From 001d78c60a69a65fb84a5b19bbbbc9aea7ee457b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:42:16 +0000 Subject: [PATCH 3/6] fix(ui): wrap SQL previews and drop the kind prefix from SQL file labels Preview lines scrolled sideways with no visible scrollbar, so long lines looked cut off; previews now wrap and the full viewer still scrolls. SQL file labels show the node name, like the graph cards. --- ui/src/lib/api/http-catalog.ts | 3 +-- ui/src/lib/build-area/labels.ts | 3 +++ ui/src/lib/components/code/code-preview.svelte | 7 ++++--- ui/src/lib/lineage/catalog-graph.ts | 3 +-- ui/src/routes/catalog/[name]/node-view.svelte | 6 +++--- ui/src/routes/catalog/[name]/overview-tab.svelte | 4 ++-- ui/src/routes/lineage/+page.svelte | 6 +++--- 7 files changed, 17 insertions(+), 15 deletions(-) diff --git a/ui/src/lib/api/http-catalog.ts b/ui/src/lib/api/http-catalog.ts index def4b6e40a..5c9c492fbd 100644 --- a/ui/src/lib/api/http-catalog.ts +++ b/ui/src/lib/api/http-catalog.ts @@ -1,4 +1,5 @@ import type { components } from '$control/schema.d.ts'; +import { shortName } from '$lib/build-area/labels'; import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, StaleReason } from './types/build'; import type { NodeKind } from './types/operate'; @@ -66,8 +67,6 @@ function staleOf(status: Wire['NodeStatus'] | null): StaleReason[] { return [{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: status.build.observed_at }]; } -/** A node key without its kind, e.g. `customer_summary` for `model:customer_summary`. */ -export const shortName = (key: string) => key.slice(key.indexOf(':') + 1); /** The listed node a key or bare name refers to, preferring a model; migrations name models without their kind. */ export function findNode(nodes: CatalogNode[], ref: string): CatalogNode | null { diff --git a/ui/src/lib/build-area/labels.ts b/ui/src/lib/build-area/labels.ts index b0bbdb1b2b..c6e5747701 100644 --- a/ui/src/lib/build-area/labels.ts +++ b/ui/src/lib/build-area/labels.ts @@ -174,3 +174,6 @@ export function relationName(relation: { database: string | null; schema: string if (!relation) return '—'; return [relation.database, relation.schema, relation.name].filter(Boolean).join('.'); } + +/** A node key without its kind, e.g. `customer_summary` for `model:customer_summary`. */ +export const shortName = (key: string) => key.slice(key.indexOf(':') + 1); diff --git a/ui/src/lib/components/code/code-preview.svelte b/ui/src/lib/components/code/code-preview.svelte index 326d87c0cd..8cd063d2fa 100644 --- a/ui/src/lib/components/code/code-preview.svelte +++ b/ui/src/lib/components/code/code-preview.svelte @@ -114,15 +114,14 @@ .ml-auto { margin-left: auto; } + /* A preview is read at a glance, so long lines wrap; the full viewer scrolls. */ .body { padding: 8px 0; - overflow-x: auto; color: var(--sb-mono); font: 12px/1.7 var(--font-mono); } .line { display: flex; - min-width: max-content; padding-right: 12px; } .ln { @@ -134,8 +133,10 @@ user-select: none; } .line code { + min-width: 0; font: inherit; - white-space: pre; + white-space: pre-wrap; + overflow-wrap: anywhere; } .more { display: block; diff --git a/ui/src/lib/lineage/catalog-graph.ts b/ui/src/lib/lineage/catalog-graph.ts index ca9ae070bf..04ff84ddda 100644 --- a/ui/src/lib/lineage/catalog-graph.ts +++ b/ui/src/lib/lineage/catalog-graph.ts @@ -2,8 +2,7 @@ import type { Edge, Node } from '@xyflow/svelte'; import type { CatalogNode, StaleReason } from '$lib/api'; -import { shortName } from '$lib/api/http-catalog'; -import { kindLine, nodeIconKey } from '$lib/build-area/labels'; +import { kindLine, nodeIconKey, shortName } from '$lib/build-area/labels'; export type CardStatus = 'ok' | 'fail' | 'warn' | 'src' | 'none'; export type CardStale = 'changed' | 'compiled' | 'upstream' | 'behind'; diff --git a/ui/src/routes/catalog/[name]/node-view.svelte b/ui/src/routes/catalog/[name]/node-view.svelte index 0e360812bd..1191e7eb7a 100644 --- a/ui/src/routes/catalog/[name]/node-view.svelte +++ b/ui/src/routes/catalog/[name]/node-view.svelte @@ -15,7 +15,7 @@ import { createResource } from '$lib/state/resource.svelte'; import { formatAgo } from '$lib/format/time'; import { nodeOutcomeLabel, nodeOutcomeTone } from '$lib/status'; - import { kindLine, nodeIcon, nodeIconKey, nodeKindPlural, sqlVersions } from '$lib/build-area/labels'; + import { kindLine, nodeIcon, nodeIconKey, nodeKindPlural, shortName, sqlVersions } from '$lib/build-area/labels'; import { cardData } from '$lib/lineage/catalog-graph'; import OverviewTab from './overview-tab.svelte'; import DriftTab from './drift-tab.svelte'; @@ -148,13 +148,13 @@
{:else if tab === 'sql'} {#if n.sql} - goto(`/catalog/${m}?tab=sql`)} /> + goto(`/catalog/${m}?tab=sql`)} /> {:else if n.code}
{/if} {:else if tab === 'drift' && n.sql} - goto(`/catalog/${m}?tab=drift`)} /> + goto(`/catalog/${m}?tab=drift`)} /> {:else if tab === 'lineage'} {#if graph.nodes.length} diff --git a/ui/src/routes/catalog/[name]/overview-tab.svelte b/ui/src/routes/catalog/[name]/overview-tab.svelte index abf7e75a32..63e874ca49 100644 --- a/ui/src/routes/catalog/[name]/overview-tab.svelte +++ b/ui/src/routes/catalog/[name]/overview-tab.svelte @@ -8,7 +8,7 @@ import { app } from '$lib/state/app.svelte'; import { formatAgo, formatStamp } from '$lib/format/time'; import { checkOutcomeLabel, checkOutcomeTone, nodeOutcomeTone, staleReasonDetail, staleReasonLabel } from '$lib/status'; - import { formatCredits, formatMs, formatNumber, kindLine, relationName, runScopePhaseLabel } from '$lib/build-area/labels'; + import { formatCredits, formatMs, formatNumber, kindLine, relationName, runScopePhaseLabel, shortName } from '$lib/build-area/labels'; import { driftLabel, driftOf, type Tab } from './utils'; let { node, onTab, onSql }: { node: CatalogNodeDetail; onTab: (tab: Tab) => void; onSql: (mode: 'raw' | 'compiled' | 'drift') => void } = $props(); @@ -75,7 +75,7 @@ {#if node.sql}
- onSql(sqlMode)} /> + onSql(sqlMode)} />
{:else if node.code}
diff --git a/ui/src/routes/lineage/+page.svelte b/ui/src/routes/lineage/+page.svelte index 18702af834..51ace19edc 100644 --- a/ui/src/routes/lineage/+page.svelte +++ b/ui/src/routes/lineage/+page.svelte @@ -38,7 +38,7 @@ import { createResource } from '$lib/state/resource.svelte'; import { formatAgo, formatStamp } from '$lib/format/time'; import { checkOutcomeTone, nodeOutcomeLabel, nodeOutcomeTone, staleReasonDetail, staleReasonLabel } from '$lib/status'; - import { NODE_KINDS, formatCredits, formatMs, formatNumber, kindLine, nodeKindPlural, relationName, runScopePhaseLabel, shortRunId, sqlVersions } from '$lib/build-area/labels'; + import { NODE_KINDS, formatCredits, formatMs, formatNumber, kindLine, nodeKindPlural, relationName, runScopePhaseLabel, shortName, shortRunId, sqlVersions } from '$lib/build-area/labels'; import { catalogGraph, cardData } from '$lib/lineage/catalog-graph'; let runOpen = $state(false); @@ -594,7 +594,7 @@ {#if sqlOpen && d?.sql}
- (sqlOpen = false)} /> + (sqlOpen = false)} />
{/if} @@ -702,7 +702,7 @@ {#if d?.sql} {#snippet meta()}{sqlRaw.split('\n').length} lines{/snippet} - openSql()} /> + openSql()} /> {:else if d?.code} From 2994b2a064db2caf1a6e7f23315f13410683f456 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:06:31 +0000 Subject: [PATCH 4/6] feat: publish column lineage and show it on catalog node pages Location publication now adds a lineage snapshot document: the compiler's FAST column lineage edges for every model, as source and target node keys, columns and transform kind. A person read returns the edges reaching or leaving one node, with each node's inferred columns, behind the same view-on-every-node gate as the catalog listing. It carries no SQL or expressions, which need view.sql. The catalog node page's Column lineage tab maps the read onto the existing column lineage view. --- clients/typescript/src/schema.d.ts | 81 +++++++++ .../authentication/password_sessions.rs | 4 + .../_helpers/resources/person_operations.rs | 27 ++- .../sqlbuild-orchestration/src/api/models.rs | 1 + .../_helpers/catalog_nodes.rs | 126 +++++++++++++- .../src/warehouse_metadata/main/mod.rs | 1 + .../main/read_person_column_lineage.rs | 51 ++++++ .../src/warehouse_metadata/models.rs | 32 ++++ .../tests/api/test_column_lineage.rs | 156 ++++++++++++++++++ .../tests/api/test_contracts.rs | 2 + .../tests/api/test_types.rs | 7 + .../bindings/_helpers/location_command/mod.rs | 1 + .../_helpers/location_command/snapshot.rs | 7 +- .../location_command/snapshot_lineage.rs | 39 +++++ .../compiler/compiled_project/_test_types.py | 7 + .../test_orchestration_snapshot.py | 54 ++++++ ui/src/lib/api/http-catalog.test.ts | 37 ++++- ui/src/lib/api/http-catalog.ts | 28 +++- ui/src/lib/api/http-client.ts | 17 +- ui/src/lib/api/types/build.ts | 2 +- .../components/lineage/col-cell-node.svelte | 5 +- ui/src/lib/lineage/column-lineage.svelte | 4 +- ui/src/routes/catalog/[name]/node-view.svelte | 2 +- 23 files changed, 676 insertions(+), 15 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 2aebecfe52..57eb4df072 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -306,6 +306,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/column-lineage": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonColumnLineage"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/connection-tests": { parameters: { query?: never; @@ -1106,6 +1122,21 @@ export interface components { CliLoginStartRequest: { client_hash: string; }; + /** @description One published column lineage edge; `transform` is the compiler's transform kind, such as `direct` or `cast`. */ + ColumnLineageEdge: { + source_column: string; + source_key: string; + target_column: string; + target_key: string; + transform: string; + }; + /** @description The column lineage reaching or leaving one node, from the current snapshot; empty before one publishes lineage. */ + ColumnLineageRead: { + edges: components["schemas"]["ColumnLineageEdge"][]; + nodes: components["schemas"]["LineageNodeColumns"][]; + /** Format: uuid */ + snapshot_id?: string | null; + }; /** @description A row whose key was already stored with different content; the first row is kept. */ ConflictEvidence: { existing_sha256: string; @@ -1236,6 +1267,16 @@ export interface components { target: string; window?: null | components["schemas"]["RunWindow"]; }; + LineageColumn: { + data_type?: string | null; + name: string; + }; + /** @description A node's columns as its compiled definition infers them; `data_type` is absent when inference could not tell. */ + LineageNodeColumns: { + columns: components["schemas"]["LineageColumn"][]; + kind: string; + node_key: string; + }; /** @description One location target the person can view, with the actions allowed on at least one of its nodes. */ LocationAccess: { actions: components["schemas"]["Action"][]; @@ -2854,6 +2895,46 @@ export interface operations { }; }; }; + readPersonColumnLineage: { + parameters: { + query: { + /** @description The node whose upstream and downstream column lineage to return */ + node_key: string; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ColumnLineageRead"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; requestConnectionTest: { parameters: { query?: never; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 5bce2d5b43..efdcea296b 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -121,6 +121,10 @@ pub(crate) fn person_routes() -> Router { "/api/control/locations/{location_id}/targets/{target}/snapshots", get(crate::api::_helpers::resources::person_reads::snapshots), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/column-lineage", + get(crate::api::_helpers::resources::person_operations::column_lineage), + ) .route( "/api/control/locations/{location_id}/targets/{target}/catalog", get(crate::api::_helpers::resources::person_operations::catalog), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index 02fcaff030..77328d67bc 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -17,12 +17,13 @@ use crate::runs::main::request_rerun::request_rerun; use crate::runs::models::{CancelRunRequest, JobRunRequest, RerunRequest, RunRecord}; use crate::store::models::ControlStore; use crate::warehouse_metadata::main::read_person_catalog::read_person_catalog; +use crate::warehouse_metadata::main::read_person_column_lineage::read_person_column_lineage; use crate::warehouse_metadata::main::read_person_fingerprint::read_person_fingerprint; use crate::warehouse_metadata::main::read_person_node_status::read_person_node_status; use crate::warehouse_metadata::main::read_person_statistics::read_person_statistics; use crate::warehouse_metadata::models::{ - CatalogPage, CatalogQuery, FingerprintObservation, FingerprintQuery, NodeStatus, - NodeStatusQuery, StatisticsPage, StatisticsQuery, StatisticsRetention, + CatalogPage, CatalogQuery, ColumnLineageRead, FingerprintObservation, FingerprintQuery, + NodeStatus, NodeStatusQuery, StatisticsPage, StatisticsQuery, StatisticsRetention, }; #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/statistics", operation_id = "readPersonStatistics", @@ -184,6 +185,28 @@ pub(crate) async fn fingerprint( )) } +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/column-lineage", operation_id = "readPersonColumnLineage", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("node_key" = String, Query, description = "The node whose upstream and downstream column lineage to return")), + responses((status = 200, body = ColumnLineageRead), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn column_lineage( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result, ApiError> { + Ok(Json( + read_person_column_lineage( + &store, + (&context, principal), + (location_id, &target, &options.node_key), + default_role, + ) + .await?, + )) +} + #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/node-status", operation_id = "readPersonNodeStatus", params(("location_id" = Uuid, Path), ("target" = String, Path), ("node_key" = String, Query, description = "One exact published node key")), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 9571345a46..7d638ea6c9 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -30,6 +30,7 @@ use crate::api::_helpers::resources::{ person_operations::statistics, person_operations::fingerprint, person_operations::node_status, + person_operations::column_lineage, person_checks::list, person_checks::history, person_checks::sql, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs index a4b23af9ff..81afc05c96 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs @@ -1,11 +1,16 @@ -//! Catalog listing rows from snapshot node definitions and each node's latest build. +//! Catalog listing rows and column lineage from snapshot node definitions and published edges. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; use serde_json::Value; use uuid::Uuid; use crate::sensors::constants::SOURCE_NODE_KIND; use crate::snapshots::models::SnapshotNode; -use crate::warehouse_metadata::models::{CatalogNodeBuild, CatalogNodeRecord, CatalogRelation}; +use crate::warehouse_metadata::models::{ + CatalogNodeBuild, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge, LineageColumn, + LineageNodeColumns, +}; /// A declared value from the definition itself, its schema entry or its config, in that order. fn declared<'a>(definition: &'a Value, key: &str) -> Option<&'a Value> { @@ -78,3 +83,120 @@ pub(crate) fn catalog_node( snapshot_id, } } + +/// Published edge rows whose both ends are current nodes, so the graph never points at a hidden node. +pub(crate) fn published_edges( + document: Option<&Value>, + nodes: &BTreeMap, +) -> Vec { + let rows = document + .and_then(|document| document.get("edges")) + .and_then(Value::as_array); + rows.into_iter() + .flatten() + .filter_map(edge_row) + .filter(|edge| nodes.contains_key(&edge.source_key) && nodes.contains_key(&edge.target_key)) + .collect() +} + +fn edge_row(row: &Value) -> Option { + let [ + source_key, + source_column, + target_key, + target_column, + transform, + ] = row.as_array()?.as_slice() + else { + return None; + }; + let text = |value: &Value| value.as_str().map(str::to_owned); + Some(ColumnLineageEdge { + source_key: text(source_key)?, + source_column: text(source_column)?, + target_key: text(target_key)?, + target_column: text(target_column)?, + transform: text(transform)?, + }) +} + +/// Edges reached upstream and downstream from any column of `node_key`, each once, in published order. +pub(crate) fn neighbourhood(edges: &[ColumnLineageEdge], node_key: &str) -> Vec { + let mut reached: BTreeSet = walk(edges, node_key, true); + reached.extend(walk(edges, node_key, false)); + reached + .into_iter() + .map(|index| edges[index].clone()) + .collect() +} + +/// The edges followed from the node's columns towards sources (`upstream`) or consumers. +fn walk(edges: &[ColumnLineageEdge], node_key: &str, upstream: bool) -> BTreeSet { + let mut reached: BTreeSet = BTreeSet::new(); + let mut by_near: HashMap<(&str, &str), Vec> = HashMap::new(); + for (index, edge) in edges.iter().enumerate() { + by_near.entry(end(edge, !upstream)).or_default().push(index); + } + let mut pending: Vec<(&str, &str)> = by_near + .keys() + .filter(|(key, _)| *key == node_key) + .copied() + .collect(); + let mut seen: BTreeSet<(&str, &str)> = pending.iter().copied().collect(); + while let Some(column) = pending.pop() { + for &index in by_near.get(&column).into_iter().flatten() { + reached.insert(index); + let far = end(&edges[index], upstream); + if seen.insert(far) { + pending.push(far); + } + } + } + reached +} + +/// An edge's source column, or its target column when `source` is false. +fn end(edge: &ColumnLineageEdge, source: bool) -> (&str, &str) { + if source { + (&edge.source_key, &edge.source_column) + } else { + (&edge.target_key, &edge.target_column) + } +} + +/// The inferred columns of the focal node and of every node an edge touches, in node key order. +pub(crate) fn node_columns( + nodes: &BTreeMap, + node_key: &str, + edges: &[ColumnLineageEdge], +) -> Vec { + let mut keys: BTreeSet<&str> = BTreeSet::from([node_key]); + for edge in edges { + keys.extend([edge.source_key.as_str(), edge.target_key.as_str()]); + } + keys.into_iter() + .filter_map(|key| nodes.get_key_value(key)) + .map(|(key, node)| LineageNodeColumns { + node_key: key.clone(), + kind: node.kind.clone(), + columns: definition_columns(&node.definition), + }) + .collect() +} + +fn definition_columns(definition: &Value) -> Vec { + let columns = definition.get("columns").and_then(Value::as_array); + columns + .into_iter() + .flatten() + .filter_map(|column| { + Some(LineageColumn { + name: column.get("name")?.as_str()?.to_owned(), + data_type: column + .get("type") + .and_then(Value::as_str) + .map(str::to_owned), + }) + }) + .collect() +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index f93dd8430f..f3f9f53dc5 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -10,6 +10,7 @@ pub mod ingest_statistics; pub mod list_person_catalog_nodes; pub(crate) mod prune_statistics; pub mod read_person_catalog; +pub mod read_person_column_lineage; pub mod read_person_fingerprint; pub mod read_person_node_status; pub mod read_person_statistics; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs new file mode 100644 index 0000000000..c03656a134 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs @@ -0,0 +1,51 @@ +//! A person reads one node's column lineage with view on every current node, as the catalog listing requires. + +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::snapshots::main::read_current_record::read_current_record; +use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::warehouse_metadata::_helpers::catalog_nodes::{ + neighbourhood, node_columns, published_edges, +}; +use crate::warehouse_metadata::models::ColumnLineageRead; + +/// Columns and edges only, never SQL or expressions, which need view.sql. +pub async fn read_person_column_lineage( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location_id, target, node_key): (Uuid, &str, &str), + default_role: DefaultRole, +) -> Result { + let (transaction, _) = authorize_listing( + begin(store, TransactionMode::Write).await?, + credentials, + (location_id, target), + default_role, + ) + .await?; + let (transaction, snapshot) = read_current_record(transaction, (location_id, target)).await?; + let Some(snapshot) = snapshot else { + commit(transaction).await?; + return Ok(ColumnLineageRead { + snapshot_id: None, + nodes: Vec::new(), + edges: Vec::new(), + }); + }; + let (transaction, nodes) = read_snapshot_nodes(transaction, snapshot.snapshot_id, None).await?; + let (transaction, documents) = + read_snapshot_documents(transaction, snapshot.snapshot_id, Some(&["lineage"])).await?; + commit(transaction).await?; + let edges = neighbourhood(&published_edges(documents.get("lineage"), &nodes), node_key); + Ok(ColumnLineageRead { + snapshot_id: Some(snapshot.snapshot_id), + nodes: node_columns(&nodes, node_key, &edges), + edges, + }) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index cf6e2589c2..e71dcc67ad 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -645,3 +645,35 @@ pub struct CatalogNodeBuild { pub run_id: String, pub version_hash: String, } + +/// A node's columns as its compiled definition infers them; `data_type` is absent when inference could not tell. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LineageNodeColumns { + pub node_key: String, + pub kind: String, + pub columns: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LineageColumn { + pub name: String, + pub data_type: Option, +} + +/// One published column lineage edge; `transform` is the compiler's transform kind, such as `direct` or `cast`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ColumnLineageEdge { + pub source_key: String, + pub source_column: String, + pub target_key: String, + pub target_column: String, + pub transform: String, +} + +/// The column lineage reaching or leaving one node, from the current snapshot; empty before one publishes lineage. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ColumnLineageRead { + pub snapshot_id: Option, + pub nodes: Vec, + pub edges: Vec, +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs b/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs new file mode 100644 index 0000000000..2af31100bf --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs @@ -0,0 +1,156 @@ +//! Published column lineage reaches people as one node's upstream and downstream columns, on both control stores. + +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::test_types::ColumnLineageTestCase; +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::snapshots::models::SnapshotNode; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::main::read_person_column_lineage::read_person_column_lineage; + +#[test] +fn given_published_column_lineage_when_people_read_a_node_then_only_its_reachable_columns_return() { + let test_cases = [ + ColumnLineageTestCase { + description: "SQLite column lineage", + backend: Backend::Sqlite, + expected_orders_edges: vec![ + ("model:raw_orders", "model:orders"), + ("model:orders", "model:order_totals"), + ], + expected_leaf_edges: 2, + }, + ColumnLineageTestCase { + description: "PostgreSQL column lineage", + backend: Backend::Postgres, + expected_orders_edges: vec![ + ("model:raw_orders", "model:orders"), + ("model:orders", "model:order_totals"), + ], + expected_leaf_edges: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let audit = AuditContext { + actor: "test-admin".into(), + granting_role: "admin".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let model = |columns: serde_json::Value| SnapshotNode { + kind: "model".into(), + definition: json!({"materialization": "table", "columns": columns}), + raw_sql: Some("select 1 as id".into()), + compiled_sql: Some("select 1 as id".into()), + python_source: None, + }; + let mut input = snapshot(location_id); + input.nodes = [ + ( + "model:raw_orders", + model(json!([{"name": "amount", "type": "VARCHAR"}])), + ), + ( + "model:orders", + model(json!([{"name": "amount", "type": "DECIMAL(10,2)"}])), + ), + ( + "model:order_totals", + model(json!([{"name": "total", "type": null}])), + ), + ( + "model:inventory", + model(json!([{"name": "quantity", "type": "INTEGER"}])), + ), + ( + "model:inventory_report", + model(json!([{"name": "quantity", "type": "INTEGER"}])), + ), + ] + .into_iter() + .map(|(key, node)| (key.to_owned(), node)) + .collect(); + input.documents.insert( + "lineage".into(), + json!({"edges": [ + ["model:raw_orders", "amount", "model:orders", "amount", "cast"], + ["model:orders", "amount", "model:order_totals", "total", "aggregate"], + ["model:inventory", "quantity", "model:inventory_report", "quantity", "direct"], + ["model:retired_orders", "amount", "model:orders", "amount", "direct"], + ]}), + ); + publish_snapshot(store, &input).await.expect("publish"); + let admin = bootstrap_user(store, &audit, "orders-admin") + .await + .expect("administrator"); + let read = |node_key: &'static str, person: Uuid| { + read_person_column_lineage( + store, + (&audit, person), + (location_id, "development", node_key), + DefaultRole::None, + ) + }; + let orders = read("model:orders", admin).await.expect("orders lineage"); + assert_eq!( + orders + .edges + .iter() + .map(|e| (e.source_key.as_str(), e.target_key.as_str())) + .collect::>(), + test_case.expected_orders_edges, + "{}: both directions, never another node's chain or a node outside the snapshot", + test_case.description + ); + assert_eq!( + json!(orders.nodes), + json!([ + {"node_key": "model:order_totals", "kind": "model", "columns": [{"name": "total", "data_type": null}]}, + {"node_key": "model:orders", "kind": "model", "columns": [{"name": "amount", "data_type": "DECIMAL(10,2)"}]}, + {"node_key": "model:raw_orders", "kind": "model", "columns": [{"name": "amount", "data_type": "VARCHAR"}]}, + ]), + "{}", + test_case.description + ); + let leaf = read("model:order_totals", admin) + .await + .expect("leaf lineage"); + assert_eq!( + leaf.edges.len(), + test_case.expected_leaf_edges, + "{}: upstream is followed transitively", + test_case.description + ); + let stranger = create_principal(store, &audit, "orders-stranger") + .await + .expect("person"); + assert!( + matches!( + read("model:orders", stranger).await, + Err(ControlError::Denied { .. }) + ), + "{}: lineage needs view on every node", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index b59038211b..584885c2a0 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -38,6 +38,8 @@ mod test_changes; mod test_checks; #[path = "test_cli_login.rs"] mod test_cli_login; +#[path = "test_column_lineage.rs"] +mod test_column_lineage; #[path = "test_contract.rs"] mod test_contract; #[path = "test_custom_roles.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 5d3d2b50f3..2ea6a1020a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -76,6 +76,13 @@ pub(crate) struct LocationWorkTestCase { pub(crate) expected_generation: i64, } +pub(crate) struct ColumnLineageTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_orders_edges: Vec<(&'static str, &'static str)>, + pub(crate) expected_leaf_edges: usize, +} + pub(crate) struct WarehouseMetadataTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index 79131a3bb0..5c4301a8d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -2,4 +2,5 @@ pub(crate) mod command; mod constants; mod runtime; mod snapshot; +mod snapshot_lineage; mod worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index e5e7028962..6e2353b734 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -19,6 +19,7 @@ use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project_roles; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::native_host::orchestration_declarations::collect_declarations; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; @@ -79,7 +80,11 @@ pub(crate) fn build_orchestration_snapshot( )?; } input = compile_project_roles(py, input, (&project_dir, graph.bind(py).get().graph()))?; - input.nodes = identity_nodes(project.into_pyobject(py)?.as_any(), input.nodes)?; + let project = project.into_pyobject(py)?; + if let Some(lineage) = column_lineage_document(project.as_any())? { + input.documents.insert("lineage".into(), lineage); + } + input.nodes = identity_nodes(project.as_any(), input.nodes)?; Ok(NativeOrchestrationSnapshot { input }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs new file mode 100644 index 0000000000..d6b65e7ef2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs @@ -0,0 +1,39 @@ +//! The project's column lineage edges, published so people can trace columns without the project. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use serde_json::{Value, json}; + +use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; +use crate::bindings::_helpers::records::construct::enum_member; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::ids::RecordId; + +/// FAST column lineage of every model as `[source key, column, target key, column, transform]` rows; `None` without SQL analysis. +pub(crate) fn column_lineage_document(project: &Bound<'_, PyAny>) -> PyResult> { + let py = project.py(); + let mode = enum_member(py, RecordId::ColumnLineageMode, "FAST")?; + let Some(lineage) = build_project_column_lineage(project, None, &mode, None)? else { + return Ok(None); + }; + let edges = field(lineage.bind(py), "edges")?; + let mut rows: Vec = Vec::new(); + for edge in edges.try_iter()? { + let edge = edge?; + let (source, target) = (field(&edge, "source")?, field(&edge, "target")?); + rows.push(json!([ + node_key(&source)?, + field(&source, "column_name")?.extract::()?, + node_key(&target)?, + field(&target, "column_name")?.extract::()?, + field(&edge, "transform_kind")?.str()?.extract::()?, + ])); + } + Ok(Some(json!({ "edges": rows }))) +} + +/// The snapshot node key of a lineage column's resource, such as `model:orders`. +fn node_key(column: &Bound<'_, PyAny>) -> PyResult { + let kind: String = field(column, "resource_type")?.str()?.extract()?; + let name: String = field(column, "resource_name")?.extract()?; + Ok(format!("{kind}:{name}")) +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 9fd65cab7a..8291e2a98c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -233,6 +233,13 @@ class SnapshotEnvironmentTestCase: expected_environment_count: int +@dataclass(frozen=True) +class SnapshotLineageTestCase: + description: str + files: dict[str, str] + expected_edges: list[list[str]] + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 7437f8ca6a..9824426203 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -35,6 +35,7 @@ SnapshotDiscoveryTestCase, SnapshotEnvironmentTestCase, SnapshotIdentityTestCase, + SnapshotLineageTestCase, SnapshotRejectionTestCase, SnapshotScheduleTestCase, ) @@ -326,6 +327,59 @@ def test_given_invalid_compilation_or_forged_nodes_when_publishing_then_no_snaps ) +@pytest.mark.parametrize( + "test_case", + [ + SnapshotLineageTestCase( + description="published column lineage follows passthroughs, casts and aggregations", + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.');\n\n" + "SELECT 1 AS order_id, 'eu' AS region, '5.00' AS amount_text\n" + ), + "models/orders.sql": ( + "MODEL (description 'Typed orders.');\n\n" + "SELECT order_id, region, CAST(amount_text AS DECIMAL(10, 2)) AS amount\n" + 'FROM __ref("stg_orders")\n' + ), + "models/region_revenue.sql": ( + "MODEL (description 'Revenue by region.');\n\n" + "SELECT region, SUM(amount) AS revenue\n" + 'FROM __ref("orders")\nGROUP BY region\n' + ), + }, + expected_edges=[ + ["model:orders", "amount", "model:region_revenue", "revenue", "aggregation"], + ["model:orders", "region", "model:region_revenue", "region", "direct"], + ["model:stg_orders", "amount_text", "model:orders", "amount", "cast"], + ["model:stg_orders", "order_id", "model:orders", "order_id", "direct"], + ["model:stg_orders", "region", "model:orders", "region", "direct"], + ], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_snapshot_document( + test_case: SnapshotLineageTestCase, + tmp_path: Path, +) -> None: + write_files(project_dir=tmp_path, files=test_case.files) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + publication: dict[str, Any] = json.loads( + build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() + ) + edges: list[list[str]] = publication["documents"]["lineage"]["edges"] + assert sorted(edges) == test_case.expected_edges + sources: set[str] = {edge[0] for edge in edges} + targets: set[str] = {edge[2] for edge in edges} + assert sources | targets <= set(publication["nodes"]) + + @pytest.mark.parametrize( "test_case", [ diff --git a/ui/src/lib/api/http-catalog.test.ts b/ui/src/lib/api/http-catalog.test.ts index 12323431a7..ef0185d1ea 100644 --- a/ui/src/lib/api/http-catalog.test.ts +++ b/ui/src/lib/api/http-catalog.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { components } from '$control/schema.d.ts'; -import { findNode, nodeDetailOf, toCatalogNodes } from './http-catalog'; +import { findNode, nodeDetailOf, toCatalogNodes, toColumnLineage } from './http-catalog'; import { toCheck } from './http-checks'; type W = components['schemas']; @@ -97,3 +97,38 @@ describe('catalog node detail', () => { expect(withSql.sql).toEqual({ raw: 'select {{ x }}', compiled: 'select {{ x }}' }); }); }); + +describe('column lineage mapping', () => { + const nodes = toCatalogNodes([node('model:raw_orders', { materialization: 'view' }), node('model:orders', { materialization: 'table' })]); + const edge = (source: string, target: string, transform: string): W['ColumnLineageEdge'] => ({ source_key: 'model:raw_orders', source_column: source, target_key: 'model:orders', target_column: target, transform }); + + it('keeps a name through direct or star as a passthrough, a new name as a rename, and anything else as a transform', () => { + const lineage = toColumnLineage( + { + snapshot_id: 's', + nodes: [ + { node_key: 'model:raw_orders', kind: 'model', columns: [{ name: 'amount', data_type: null }] }, + { node_key: 'model:orders', kind: 'model', columns: [{ name: 'amount', data_type: 'DECIMAL(10,2)' }] }, + { node_key: 'model:retired', kind: 'model', columns: [] } + ], + edges: [edge('id', 'id', 'direct'), edge('note', 'note', 'star'), edge('cust', 'customer_id', 'direct'), edge('amount', 'amount', 'cast')] + }, + nodes + ); + expect(lineage?.edges.map(([, target, lens]) => [target, lens])).toEqual([ + ['model:orders.id', 'passthrough'], + ['model:orders.note', 'passthrough'], + ['model:orders.customer_id', 'rename'], + ['model:orders.amount', 'transform'] + ]); + expect(lineage?.models.map((m) => [m.id, m.name, m.kind, m.columns])).toEqual([ + ['model:raw_orders', 'raw_orders', 'view', [{ name: 'amount', type: '' }]], + ['model:orders', 'orders', 'table', [{ name: 'amount', type: 'DECIMAL(10,2)' }]] + ]); + expect([lineage?.sql, lineage?.expressions]).toEqual([{}, {}]); + }); + + it('has nothing to show when no lineage reaches the node', () => { + expect(toColumnLineage({ snapshot_id: 's', nodes: [], edges: [] }, nodes)).toBeNull(); + }); +}); diff --git a/ui/src/lib/api/http-catalog.ts b/ui/src/lib/api/http-catalog.ts index 5c9c492fbd..63ba2d759a 100644 --- a/ui/src/lib/api/http-catalog.ts +++ b/ui/src/lib/api/http-catalog.ts @@ -1,6 +1,7 @@ import type { components } from '$control/schema.d.ts'; import { shortName } from '$lib/build-area/labels'; -import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, StaleReason } from './types/build'; +import { cardStatus } from '$lib/lineage/catalog-graph'; +import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, ColumnLens, ColumnLineage, StaleReason } from './types/build'; import type { NodeKind } from './types/operate'; type Wire = components['schemas']; @@ -104,3 +105,28 @@ export function nodeDetailOf(nodes: CatalogNode[], node: CatalogNode, status: Wi history: null }; } + +/** A kept name through `direct` or `star` is a passthrough, a new name a rename; every other transform changes the value. */ +function lensOf(edge: Wire['ColumnLineageEdge']): ColumnLens { + if (edge.transform !== 'direct' && edge.transform !== 'star') return 'transform'; + return edge.source_column === edge.target_column ? 'passthrough' : 'rename'; +} + +/** + * Column lineage around one node, named and coloured from the catalog listing. The read carries no + * SQL or expressions, so those stay empty; null before a snapshot publishes lineage for the node. + */ +export function toColumnLineage(read: Wire['ColumnLineageRead'], nodes: CatalogNode[]): ColumnLineage | null { + if (!read.edges.length) return null; + const listed = new Map(nodes.map((n) => [n.key, n])); + return { + models: read.nodes.flatMap((n) => { + const node = listed.get(n.node_key); + const columns = n.columns.map((c) => ({ name: c.name, type: c.data_type ?? '' })); + return node ? [{ id: n.node_key, name: shortName(n.node_key), kind: node.kind, status: cardStatus(node), columns }] : []; + }), + edges: read.edges.map((e) => [`${e.source_key}.${e.source_column}`, `${e.target_key}.${e.target_column}`, lensOf(e)]), + sql: {}, + expressions: {} + }; +} diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 952caa2133..6600d53438 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -2,13 +2,13 @@ import createFetch from 'openapi-fetch'; import type { components, paths } from '$control/schema.d.ts'; import { ApiError, type ApiClient } from './client'; import { checkDetailOf, toCheck, toCheckEvidence, toCheckSql } from './http-checks'; -import { findNode, nodeDetailOf, toCatalogNodes } from './http-catalog'; +import { findNode, nodeDetailOf, toCatalogNodes, toColumnLineage } from './http-catalog'; import { toSnapshotChanges } from './http-changes'; import { jobDetailOf, toJob, toTick, toTrigger, triggerDetailOf } from './http-jobs'; import { RUN_WINDOW, runDetailOf, runRowPage, runsMatching, runTabCountsOf, toRunNode, toRuns } from './http-runs'; import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals'; import type { AuthMode, Bootstrap, Capability, Location, Scope } from './types'; -import type { CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, SnapshotChange } from './types/build'; +import type { CatalogNode, CatalogNodeDetail, Check, ColumnLineage, CheckDetail, CheckEvidence, CheckSql, SnapshotChange } from './types/build'; import type { Job, JobDetail, RerunMode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState } from './types/operate'; type Wire = components['schemas']; @@ -39,6 +39,7 @@ const SUPPORTED: ReadonlySet = new Set([ 'getCheckEvidence', 'listCatalogNodes', 'getCatalogNode', + 'getColumnLineage', 'listSnapshotChanges' ]); @@ -263,6 +264,18 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt if (!snapshots.data) throw new ApiError(errorText(snapshots.error, snapshots.response), snapshots.response.status); return toSnapshotChanges(snapshots.data, runs); }, + async getColumnLineage(scope: Scope, ref: string): Promise { + const nodes = await implemented.listCatalogNodes(scope); + const node = findNode(nodes, ref); + if (!node) return null; + const { locationId, target } = parseLocationKey(scope.locationId); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/column-lineage', { + params: { path: { location_id: locationId, target }, query: { node_key: node.key } } + }); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return toColumnLineage(data, nodes); + }, async getCatalogNode(scope: Scope, ref: string): Promise { const nodes = await implemented.listCatalogNodes(scope); const node = findNode(nodes, ref); diff --git a/ui/src/lib/api/types/build.ts b/ui/src/lib/api/types/build.ts index 906f9e77df..e9b0e51bb9 100644 --- a/ui/src/lib/api/types/build.ts +++ b/ui/src/lib/api/types/build.ts @@ -193,7 +193,7 @@ export type CatalogNodeDetail = Omit & { export type ColumnLens = 'raw' | 'passthrough' | 'rename' | 'transform'; export type ColumnLineage = { - models: { id: string; name: string; kind: string; status: 'ok' | 'fail' | 'warn' | 'stale' | 'src'; columns: { name: string; type: string }[] }[]; + models: { id: string; name: string; kind: string; status: 'ok' | 'fail' | 'warn' | 'stale' | 'src' | 'none'; columns: { name: string; type: string }[] }[]; /** `model.column` pairs from upstream to downstream. */ edges: [string, string, ColumnLens][]; sql: Record; diff --git a/ui/src/lib/components/lineage/col-cell-node.svelte b/ui/src/lib/components/lineage/col-cell-node.svelte index c1f24b1380..7b0e7752b4 100644 --- a/ui/src/lib/components/lineage/col-cell-node.svelte +++ b/ui/src/lib/components/lineage/col-cell-node.svelte @@ -7,7 +7,7 @@ model: string; column: string; kind: string; - status: 'ok' | 'fail' | 'warn' | 'stale' | 'src'; + status: 'ok' | 'fail' | 'warn' | 'stale' | 'src' | 'none'; lens?: Lens; lensLabel?: Record; focal?: boolean; @@ -22,7 +22,8 @@ fail: 'var(--sb-error)', warn: 'var(--sb-warning)', stale: 'var(--sb-stale)', - src: 'var(--sb-text-faint)' + src: 'var(--sb-text-faint)', + none: 'var(--border)' }; const KindIcon = $derived(d.kind === 'view' ? Eye : d.kind.startsWith('source') ? Database : d.kind.startsWith('incremental') ? Layers : Table2); const kindColor = $derived(d.kind === 'view' ? '#5aa9e6' : d.kind === 'table' ? '#2e90ff' : d.kind.startsWith('incremental') ? '#3fb6c9' : '#7c8aa5'); diff --git a/ui/src/lib/lineage/column-lineage.svelte b/ui/src/lib/lineage/column-lineage.svelte index b418a1ab07..6ec3e4780d 100644 --- a/ui/src/lib/lineage/column-lineage.svelte +++ b/ui/src/lib/lineage/column-lineage.svelte @@ -12,7 +12,7 @@ import { highlightSqlbuild } from '$lib/sql/highlight'; import { tick } from 'svelte'; - type ColModel = { id: string; name: string; kind: string; status: 'ok' | 'fail' | 'warn' | 'stale' | 'src'; columns: { name: string; type: string }[] }; + type ColModel = { id: string; name: string; kind: string; status: 'ok' | 'fail' | 'warn' | 'stale' | 'src' | 'none'; columns: { name: string; type: string }[] }; type Lens = 'raw' | 'passthrough' | 'rename' | 'transform'; let { @@ -394,7 +394,7 @@ {#if sqlParts && pinnedNode}
SQL · {sqlParts.model}.sql
{@html sqlParts.before}{#if sqlParts.match}{@html sqlParts.match}{/if}{@html sqlParts.after}
- {:else} + {:else if modelSql && Object.keys(modelSql).length}

Select a step to see it in the model's SQL.

{/if} diff --git a/ui/src/routes/catalog/[name]/node-view.svelte b/ui/src/routes/catalog/[name]/node-view.svelte index 1191e7eb7a..2af4eca3c3 100644 --- a/ui/src/routes/catalog/[name]/node-view.svelte +++ b/ui/src/routes/catalog/[name]/node-view.svelte @@ -97,7 +97,7 @@
-

{name}

+

{shortName(n?.key ?? name)}

{#if n} {#if n.kind === 'source'} From 2c59909a75708887190872efcc833e2a93e3414b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:10:36 +0000 Subject: [PATCH 5/6] fix(ui): show full names and transform kinds in column lineage Traced identifiers and node names wrap instead of truncating, column types stay on one line, and transformed edges name the compiler's transform kind, such as Cast or Aggregation. --- ui/src/lib/api/http-catalog.test.ts | 12 ++++---- ui/src/lib/api/http-catalog.ts | 8 ++++- ui/src/lib/api/types/build.ts | 4 +-- .../components/lineage/col-cell-node.svelte | 16 +++++----- ui/src/lib/lineage/column-lineage.svelte | 30 +++++++++++-------- 5 files changed, 42 insertions(+), 28 deletions(-) diff --git a/ui/src/lib/api/http-catalog.test.ts b/ui/src/lib/api/http-catalog.test.ts index ef0185d1ea..398299bc76 100644 --- a/ui/src/lib/api/http-catalog.test.ts +++ b/ui/src/lib/api/http-catalog.test.ts @@ -102,7 +102,7 @@ describe('column lineage mapping', () => { const nodes = toCatalogNodes([node('model:raw_orders', { materialization: 'view' }), node('model:orders', { materialization: 'table' })]); const edge = (source: string, target: string, transform: string): W['ColumnLineageEdge'] => ({ source_key: 'model:raw_orders', source_column: source, target_key: 'model:orders', target_column: target, transform }); - it('keeps a name through direct or star as a passthrough, a new name as a rename, and anything else as a transform', () => { + it('keeps a name through direct or star as a passthrough, a new name as a rename, and names any other transform', () => { const lineage = toColumnLineage( { snapshot_id: 's', @@ -115,11 +115,11 @@ describe('column lineage mapping', () => { }, nodes ); - expect(lineage?.edges.map(([, target, lens]) => [target, lens])).toEqual([ - ['model:orders.id', 'passthrough'], - ['model:orders.note', 'passthrough'], - ['model:orders.customer_id', 'rename'], - ['model:orders.amount', 'transform'] + expect(lineage?.edges.map(([, target, lens, label]) => [target, lens, label])).toEqual([ + ['model:orders.id', 'passthrough', undefined], + ['model:orders.note', 'passthrough', undefined], + ['model:orders.customer_id', 'rename', undefined], + ['model:orders.amount', 'transform', 'Cast'] ]); expect(lineage?.models.map((m) => [m.id, m.name, m.kind, m.columns])).toEqual([ ['model:raw_orders', 'raw_orders', 'view', [{ name: 'amount', type: '' }]], diff --git a/ui/src/lib/api/http-catalog.ts b/ui/src/lib/api/http-catalog.ts index 63ba2d759a..76b50cb991 100644 --- a/ui/src/lib/api/http-catalog.ts +++ b/ui/src/lib/api/http-catalog.ts @@ -112,6 +112,12 @@ function lensOf(edge: Wire['ColumnLineageEdge']): ColumnLens { return edge.source_column === edge.target_column ? 'passthrough' : 'rename'; } +/** The compiler's transform kind as a label, such as `Cast`; passthroughs and renames keep the lens's own name. */ +function transformLabel(edge: Wire['ColumnLineageEdge']): string | undefined { + if (lensOf(edge) !== 'transform') return undefined; + return edge.transform === 'unknown' ? 'Transformed' : edge.transform.charAt(0).toUpperCase() + edge.transform.slice(1); +} + /** * Column lineage around one node, named and coloured from the catalog listing. The read carries no * SQL or expressions, so those stay empty; null before a snapshot publishes lineage for the node. @@ -125,7 +131,7 @@ export function toColumnLineage(read: Wire['ColumnLineageRead'], nodes: CatalogN const columns = n.columns.map((c) => ({ name: c.name, type: c.data_type ?? '' })); return node ? [{ id: n.node_key, name: shortName(n.node_key), kind: node.kind, status: cardStatus(node), columns }] : []; }), - edges: read.edges.map((e) => [`${e.source_key}.${e.source_column}`, `${e.target_key}.${e.target_column}`, lensOf(e)]), + edges: read.edges.map((e) => [`${e.source_key}.${e.source_column}`, `${e.target_key}.${e.target_column}`, lensOf(e), transformLabel(e)]), sql: {}, expressions: {} }; diff --git a/ui/src/lib/api/types/build.ts b/ui/src/lib/api/types/build.ts index e9b0e51bb9..81e534b63f 100644 --- a/ui/src/lib/api/types/build.ts +++ b/ui/src/lib/api/types/build.ts @@ -194,8 +194,8 @@ export type ColumnLens = 'raw' | 'passthrough' | 'rename' | 'transform'; export type ColumnLineage = { models: { id: string; name: string; kind: string; status: 'ok' | 'fail' | 'warn' | 'stale' | 'src' | 'none'; columns: { name: string; type: string }[] }[]; - /** `model.column` pairs from upstream to downstream. */ - edges: [string, string, ColumnLens][]; + /** `model.column` pairs from upstream to downstream, with an optional label naming the transform, such as `Cast`. */ + edges: [string, string, ColumnLens, string?][]; sql: Record; /** The defining expression of each column, per model. */ expressions: Record>; diff --git a/ui/src/lib/components/lineage/col-cell-node.svelte b/ui/src/lib/components/lineage/col-cell-node.svelte index 7b0e7752b4..0ed67190b9 100644 --- a/ui/src/lib/components/lineage/col-cell-node.svelte +++ b/ui/src/lib/components/lineage/col-cell-node.svelte @@ -9,7 +9,8 @@ kind: string; status: 'ok' | 'fail' | 'warn' | 'stale' | 'src' | 'none'; lens?: Lens; - lensLabel?: Record; + /** What the incoming edge does, such as `Cast`. */ + label?: string; focal?: boolean; active?: boolean; }; @@ -32,21 +33,22 @@
- {d.column} + {d.column}
- - {d.model} - {#if d.lens && d.lensLabel} - {d.lensLabel[d.lens]} + + {d.model} + {#if d.lens && d.label} + {d.label} {:else} {d.kind} {/if} diff --git a/ui/src/lib/lineage/column-lineage.svelte b/ui/src/lib/lineage/column-lineage.svelte index 6ec3e4780d..00096734b7 100644 --- a/ui/src/lib/lineage/column-lineage.svelte +++ b/ui/src/lib/lineage/column-lineage.svelte @@ -24,7 +24,7 @@ }: { focal: string; models: ColModel[]; - colEdges: [string, string, Lens][]; + colEdges: [string, string, Lens, string?][]; /** per-model SQL: modelId -> SQL text */ modelSql?: Record; /** per-model column expressions: modelId -> { colName -> defining substring } */ @@ -70,6 +70,9 @@ const kindOf = (id: string) => models.find((m) => m.id === id)?.kind ?? 'table'; const statusOf = (id: string) => models.find((m) => m.id === id)?.status ?? 'ok'; const incomingLens = $derived(new Map(colEdges.map(([, t, lens]) => [t, lens]))); + // What a column's incoming edge does: the published transform, such as Cast, or the lens's name. + const incomingLabel = $derived(new Map(colEdges.map(([, t, lens, label]) => [t, label ?? lensLabel[lens]]))); + const labelOf = (cid: string) => incomingLabel.get(cid) ?? lensLabel.raw; const focalModel = $derived(models.find((m) => m.id === focal)); let colQuery = $state(''); @@ -223,7 +226,7 @@ kind: kindOf(modelOf(cid)), status: statusOf(modelOf(cid)), lens: incomingLens.get(cid), - lensLabel, + label: incomingLens.has(cid) ? labelOf(cid) : undefined, focal: cid === selectedCol, active: cid === activeStep } @@ -284,9 +287,9 @@ onclick={() => (selectedCol = cid)} > {#if selectedCol === cid}{/if} - {c.name} - {lens ? lensLabel[lens] : ''} - {c.type.toLowerCase()} + {c.name} + {lens ? labelOf(cid) : ''} + {#if c.type}{c.type.toLowerCase()}{/if} {:else}
No columns match.
@@ -369,8 +372,8 @@
    • @@ -383,8 +386,8 @@
    • @@ -498,11 +501,14 @@ border-color: var(--primary); background: var(--primary); } + /* The identifier being traced wraps rather than truncates. */ .dv-name { - overflow: hidden; font: 12.5px var(--font-mono); - text-overflow: ellipsis; - white-space: nowrap; + overflow-wrap: anywhere; + } + .col-name { + min-width: 0; + overflow-wrap: anywhere; } .dv-name .m { color: var(--muted-foreground); From 2ff5a9ea79640ca535845b8f9846ef8b7d145bd1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:15:57 +0000 Subject: [PATCH 6/6] refactor: move column lineage to its own change Column lineage publication would push large projects past the publish body limit, so it lands after the publish request is compressed. This change keeps node detail, labels and the SQL preview fixes. --- clients/typescript/src/schema.d.ts | 81 --------- .../authentication/password_sessions.rs | 4 - .../_helpers/resources/person_operations.rs | 27 +-- .../sqlbuild-orchestration/src/api/models.rs | 1 - .../_helpers/catalog_nodes.rs | 126 +------------- .../src/warehouse_metadata/main/mod.rs | 1 - .../main/read_person_column_lineage.rs | 51 ------ .../src/warehouse_metadata/models.rs | 32 ---- .../tests/api/test_column_lineage.rs | 156 ------------------ .../tests/api/test_contracts.rs | 2 - .../tests/api/test_types.rs | 7 - .../bindings/_helpers/location_command/mod.rs | 1 - .../_helpers/location_command/snapshot.rs | 7 +- .../location_command/snapshot_lineage.rs | 39 ----- .../compiler/compiled_project/_test_types.py | 7 - .../test_orchestration_snapshot.py | 54 ------ ui/src/lib/api/http-catalog.test.ts | 37 +---- ui/src/lib/api/http-catalog.ts | 34 +--- ui/src/lib/api/http-client.ts | 17 +- ui/src/lib/api/types/build.ts | 6 +- .../components/lineage/col-cell-node.svelte | 21 +-- ui/src/lib/lineage/column-lineage.svelte | 34 ++-- 22 files changed, 35 insertions(+), 710 deletions(-) delete mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs delete mode 100644 crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 57eb4df072..2aebecfe52 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -306,22 +306,6 @@ export interface paths { patch?: never; trace?: never; }; - "/api/control/locations/{location_id}/targets/{target}/column-lineage": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get: operations["readPersonColumnLineage"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/api/control/locations/{location_id}/targets/{target}/connection-tests": { parameters: { query?: never; @@ -1122,21 +1106,6 @@ export interface components { CliLoginStartRequest: { client_hash: string; }; - /** @description One published column lineage edge; `transform` is the compiler's transform kind, such as `direct` or `cast`. */ - ColumnLineageEdge: { - source_column: string; - source_key: string; - target_column: string; - target_key: string; - transform: string; - }; - /** @description The column lineage reaching or leaving one node, from the current snapshot; empty before one publishes lineage. */ - ColumnLineageRead: { - edges: components["schemas"]["ColumnLineageEdge"][]; - nodes: components["schemas"]["LineageNodeColumns"][]; - /** Format: uuid */ - snapshot_id?: string | null; - }; /** @description A row whose key was already stored with different content; the first row is kept. */ ConflictEvidence: { existing_sha256: string; @@ -1267,16 +1236,6 @@ export interface components { target: string; window?: null | components["schemas"]["RunWindow"]; }; - LineageColumn: { - data_type?: string | null; - name: string; - }; - /** @description A node's columns as its compiled definition infers them; `data_type` is absent when inference could not tell. */ - LineageNodeColumns: { - columns: components["schemas"]["LineageColumn"][]; - kind: string; - node_key: string; - }; /** @description One location target the person can view, with the actions allowed on at least one of its nodes. */ LocationAccess: { actions: components["schemas"]["Action"][]; @@ -2895,46 +2854,6 @@ export interface operations { }; }; }; - readPersonColumnLineage: { - parameters: { - query: { - /** @description The node whose upstream and downstream column lineage to return */ - node_key: string; - }; - header?: never; - path: { - location_id: string; - target: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["ColumnLineageRead"]; - }; - }; - /** @description Session rejected */ - 401: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["ApiErrorBody"]; - }; - }; - }; - }; requestConnectionTest: { parameters: { query?: never; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index efdcea296b..5bce2d5b43 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -121,10 +121,6 @@ pub(crate) fn person_routes() -> Router { "/api/control/locations/{location_id}/targets/{target}/snapshots", get(crate::api::_helpers::resources::person_reads::snapshots), ) - .route( - "/api/control/locations/{location_id}/targets/{target}/column-lineage", - get(crate::api::_helpers::resources::person_operations::column_lineage), - ) .route( "/api/control/locations/{location_id}/targets/{target}/catalog", get(crate::api::_helpers::resources::person_operations::catalog), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index 77328d67bc..02fcaff030 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -17,13 +17,12 @@ use crate::runs::main::request_rerun::request_rerun; use crate::runs::models::{CancelRunRequest, JobRunRequest, RerunRequest, RunRecord}; use crate::store::models::ControlStore; use crate::warehouse_metadata::main::read_person_catalog::read_person_catalog; -use crate::warehouse_metadata::main::read_person_column_lineage::read_person_column_lineage; use crate::warehouse_metadata::main::read_person_fingerprint::read_person_fingerprint; use crate::warehouse_metadata::main::read_person_node_status::read_person_node_status; use crate::warehouse_metadata::main::read_person_statistics::read_person_statistics; use crate::warehouse_metadata::models::{ - CatalogPage, CatalogQuery, ColumnLineageRead, FingerprintObservation, FingerprintQuery, - NodeStatus, NodeStatusQuery, StatisticsPage, StatisticsQuery, StatisticsRetention, + CatalogPage, CatalogQuery, FingerprintObservation, FingerprintQuery, NodeStatus, + NodeStatusQuery, StatisticsPage, StatisticsQuery, StatisticsRetention, }; #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/statistics", operation_id = "readPersonStatistics", @@ -185,28 +184,6 @@ pub(crate) async fn fingerprint( )) } -#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/column-lineage", operation_id = "readPersonColumnLineage", - params(("location_id" = Uuid, Path), ("target" = String, Path), - ("node_key" = String, Query, description = "The node whose upstream and downstream column lineage to return")), - responses((status = 200, body = ColumnLineageRead), (status = 401, description = "Session rejected"), - (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] -pub(crate) async fn column_lineage( - (State(store), Extension(context)): (State, Extension), - Extension(default_role): Extension, - (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), - Extension(AuthenticatedPerson(principal, _)): Extension, -) -> Result, ApiError> { - Ok(Json( - read_person_column_lineage( - &store, - (&context, principal), - (location_id, &target, &options.node_key), - default_role, - ) - .await?, - )) -} - #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/node-status", operation_id = "readPersonNodeStatus", params(("location_id" = Uuid, Path), ("target" = String, Path), ("node_key" = String, Query, description = "One exact published node key")), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 7d638ea6c9..9571345a46 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -30,7 +30,6 @@ use crate::api::_helpers::resources::{ person_operations::statistics, person_operations::fingerprint, person_operations::node_status, - person_operations::column_lineage, person_checks::list, person_checks::history, person_checks::sql, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs index 81afc05c96..a4b23af9ff 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs @@ -1,16 +1,11 @@ -//! Catalog listing rows and column lineage from snapshot node definitions and published edges. - -use std::collections::{BTreeMap, BTreeSet, HashMap}; +//! Catalog listing rows from snapshot node definitions and each node's latest build. use serde_json::Value; use uuid::Uuid; use crate::sensors::constants::SOURCE_NODE_KIND; use crate::snapshots::models::SnapshotNode; -use crate::warehouse_metadata::models::{ - CatalogNodeBuild, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge, LineageColumn, - LineageNodeColumns, -}; +use crate::warehouse_metadata::models::{CatalogNodeBuild, CatalogNodeRecord, CatalogRelation}; /// A declared value from the definition itself, its schema entry or its config, in that order. fn declared<'a>(definition: &'a Value, key: &str) -> Option<&'a Value> { @@ -83,120 +78,3 @@ pub(crate) fn catalog_node( snapshot_id, } } - -/// Published edge rows whose both ends are current nodes, so the graph never points at a hidden node. -pub(crate) fn published_edges( - document: Option<&Value>, - nodes: &BTreeMap, -) -> Vec { - let rows = document - .and_then(|document| document.get("edges")) - .and_then(Value::as_array); - rows.into_iter() - .flatten() - .filter_map(edge_row) - .filter(|edge| nodes.contains_key(&edge.source_key) && nodes.contains_key(&edge.target_key)) - .collect() -} - -fn edge_row(row: &Value) -> Option { - let [ - source_key, - source_column, - target_key, - target_column, - transform, - ] = row.as_array()?.as_slice() - else { - return None; - }; - let text = |value: &Value| value.as_str().map(str::to_owned); - Some(ColumnLineageEdge { - source_key: text(source_key)?, - source_column: text(source_column)?, - target_key: text(target_key)?, - target_column: text(target_column)?, - transform: text(transform)?, - }) -} - -/// Edges reached upstream and downstream from any column of `node_key`, each once, in published order. -pub(crate) fn neighbourhood(edges: &[ColumnLineageEdge], node_key: &str) -> Vec { - let mut reached: BTreeSet = walk(edges, node_key, true); - reached.extend(walk(edges, node_key, false)); - reached - .into_iter() - .map(|index| edges[index].clone()) - .collect() -} - -/// The edges followed from the node's columns towards sources (`upstream`) or consumers. -fn walk(edges: &[ColumnLineageEdge], node_key: &str, upstream: bool) -> BTreeSet { - let mut reached: BTreeSet = BTreeSet::new(); - let mut by_near: HashMap<(&str, &str), Vec> = HashMap::new(); - for (index, edge) in edges.iter().enumerate() { - by_near.entry(end(edge, !upstream)).or_default().push(index); - } - let mut pending: Vec<(&str, &str)> = by_near - .keys() - .filter(|(key, _)| *key == node_key) - .copied() - .collect(); - let mut seen: BTreeSet<(&str, &str)> = pending.iter().copied().collect(); - while let Some(column) = pending.pop() { - for &index in by_near.get(&column).into_iter().flatten() { - reached.insert(index); - let far = end(&edges[index], upstream); - if seen.insert(far) { - pending.push(far); - } - } - } - reached -} - -/// An edge's source column, or its target column when `source` is false. -fn end(edge: &ColumnLineageEdge, source: bool) -> (&str, &str) { - if source { - (&edge.source_key, &edge.source_column) - } else { - (&edge.target_key, &edge.target_column) - } -} - -/// The inferred columns of the focal node and of every node an edge touches, in node key order. -pub(crate) fn node_columns( - nodes: &BTreeMap, - node_key: &str, - edges: &[ColumnLineageEdge], -) -> Vec { - let mut keys: BTreeSet<&str> = BTreeSet::from([node_key]); - for edge in edges { - keys.extend([edge.source_key.as_str(), edge.target_key.as_str()]); - } - keys.into_iter() - .filter_map(|key| nodes.get_key_value(key)) - .map(|(key, node)| LineageNodeColumns { - node_key: key.clone(), - kind: node.kind.clone(), - columns: definition_columns(&node.definition), - }) - .collect() -} - -fn definition_columns(definition: &Value) -> Vec { - let columns = definition.get("columns").and_then(Value::as_array); - columns - .into_iter() - .flatten() - .filter_map(|column| { - Some(LineageColumn { - name: column.get("name")?.as_str()?.to_owned(), - data_type: column - .get("type") - .and_then(Value::as_str) - .map(str::to_owned), - }) - }) - .collect() -} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index f3f9f53dc5..f93dd8430f 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -10,7 +10,6 @@ pub mod ingest_statistics; pub mod list_person_catalog_nodes; pub(crate) mod prune_statistics; pub mod read_person_catalog; -pub mod read_person_column_lineage; pub mod read_person_fingerprint; pub mod read_person_node_status; pub mod read_person_statistics; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs deleted file mode 100644 index c03656a134..0000000000 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/read_person_column_lineage.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! A person reads one node's column lineage with view on every current node, as the catalog listing requires. - -use uuid::Uuid; - -use crate::authorization::main::authorize_listing::authorize_listing; -use crate::credentials::models::{AuditContext, DefaultRole}; -use crate::snapshots::main::read_current_record::read_current_record; -use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents; -use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; -use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit}; -use crate::store::models::{ControlStore, TransactionMode}; -use crate::warehouse_metadata::_helpers::catalog_nodes::{ - neighbourhood, node_columns, published_edges, -}; -use crate::warehouse_metadata::models::ColumnLineageRead; - -/// Columns and edges only, never SQL or expressions, which need view.sql. -pub async fn read_person_column_lineage( - store: &ControlStore, - credentials: (&AuditContext, Uuid), - (location_id, target, node_key): (Uuid, &str, &str), - default_role: DefaultRole, -) -> Result { - let (transaction, _) = authorize_listing( - begin(store, TransactionMode::Write).await?, - credentials, - (location_id, target), - default_role, - ) - .await?; - let (transaction, snapshot) = read_current_record(transaction, (location_id, target)).await?; - let Some(snapshot) = snapshot else { - commit(transaction).await?; - return Ok(ColumnLineageRead { - snapshot_id: None, - nodes: Vec::new(), - edges: Vec::new(), - }); - }; - let (transaction, nodes) = read_snapshot_nodes(transaction, snapshot.snapshot_id, None).await?; - let (transaction, documents) = - read_snapshot_documents(transaction, snapshot.snapshot_id, Some(&["lineage"])).await?; - commit(transaction).await?; - let edges = neighbourhood(&published_edges(documents.get("lineage"), &nodes), node_key); - Ok(ColumnLineageRead { - snapshot_id: Some(snapshot.snapshot_id), - nodes: node_columns(&nodes, node_key, &edges), - edges, - }) -} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index e71dcc67ad..cf6e2589c2 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -645,35 +645,3 @@ pub struct CatalogNodeBuild { pub run_id: String, pub version_hash: String, } - -/// A node's columns as its compiled definition infers them; `data_type` is absent when inference could not tell. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] -pub struct LineageNodeColumns { - pub node_key: String, - pub kind: String, - pub columns: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] -pub struct LineageColumn { - pub name: String, - pub data_type: Option, -} - -/// One published column lineage edge; `transform` is the compiler's transform kind, such as `direct` or `cast`. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] -pub struct ColumnLineageEdge { - pub source_key: String, - pub source_column: String, - pub target_key: String, - pub target_column: String, - pub transform: String, -} - -/// The column lineage reaching or leaving one node, from the current snapshot; empty before one publishes lineage. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] -pub struct ColumnLineageRead { - pub snapshot_id: Option, - pub nodes: Vec, - pub edges: Vec, -} diff --git a/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs b/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs deleted file mode 100644 index 2af31100bf..0000000000 --- a/crates/sqlbuild-orchestration/tests/api/test_column_lineage.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! Published column lineage reaches people as one node's upstream and downstream columns, on both control stores. - -use serde_json::json; -use uuid::Uuid; - -use crate::api::tests::test_types::ColumnLineageTestCase; -use crate::authorization::main::bootstrap_user::bootstrap_user; -use crate::credentials::main::identities::create_principal::create_principal; -use crate::credentials::models::{AuditContext, DefaultRole}; -use crate::credentials::tests::location_helpers::create_location; -use crate::runs::tests::helpers::snapshot; -use crate::snapshots::main::publish_snapshot::publish_snapshot; -use crate::snapshots::models::SnapshotNode; -use crate::store::errors::ControlError; -use crate::store::main::migrate_store::migrate_store; -use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; -use crate::store::tests::store_test_types::Backend; -use crate::warehouse_metadata::main::read_person_column_lineage::read_person_column_lineage; - -#[test] -fn given_published_column_lineage_when_people_read_a_node_then_only_its_reachable_columns_return() { - let test_cases = [ - ColumnLineageTestCase { - description: "SQLite column lineage", - backend: Backend::Sqlite, - expected_orders_edges: vec![ - ("model:raw_orders", "model:orders"), - ("model:orders", "model:order_totals"), - ], - expected_leaf_edges: 2, - }, - ColumnLineageTestCase { - description: "PostgreSQL column lineage", - backend: Backend::Postgres, - expected_orders_edges: vec![ - ("model:raw_orders", "model:orders"), - ("model:orders", "model:order_totals"), - ], - expected_leaf_edges: 2, - }, - ]; - for test_case in test_cases { - run(async { - let Some(fixture) = open_test_store(test_case.backend).await else { - return; - }; - let store = &fixture.store; - migrate_store(store).await.expect("migrate"); - let audit = AuditContext { - actor: "test-admin".into(), - granting_role: "admin".into(), - request_id: Uuid::now_v7(), - ip_address: None, - }; - let location_id = create_location(store, "orders", "test-admin") - .await - .expect("location"); - let model = |columns: serde_json::Value| SnapshotNode { - kind: "model".into(), - definition: json!({"materialization": "table", "columns": columns}), - raw_sql: Some("select 1 as id".into()), - compiled_sql: Some("select 1 as id".into()), - python_source: None, - }; - let mut input = snapshot(location_id); - input.nodes = [ - ( - "model:raw_orders", - model(json!([{"name": "amount", "type": "VARCHAR"}])), - ), - ( - "model:orders", - model(json!([{"name": "amount", "type": "DECIMAL(10,2)"}])), - ), - ( - "model:order_totals", - model(json!([{"name": "total", "type": null}])), - ), - ( - "model:inventory", - model(json!([{"name": "quantity", "type": "INTEGER"}])), - ), - ( - "model:inventory_report", - model(json!([{"name": "quantity", "type": "INTEGER"}])), - ), - ] - .into_iter() - .map(|(key, node)| (key.to_owned(), node)) - .collect(); - input.documents.insert( - "lineage".into(), - json!({"edges": [ - ["model:raw_orders", "amount", "model:orders", "amount", "cast"], - ["model:orders", "amount", "model:order_totals", "total", "aggregate"], - ["model:inventory", "quantity", "model:inventory_report", "quantity", "direct"], - ["model:retired_orders", "amount", "model:orders", "amount", "direct"], - ]}), - ); - publish_snapshot(store, &input).await.expect("publish"); - let admin = bootstrap_user(store, &audit, "orders-admin") - .await - .expect("administrator"); - let read = |node_key: &'static str, person: Uuid| { - read_person_column_lineage( - store, - (&audit, person), - (location_id, "development", node_key), - DefaultRole::None, - ) - }; - let orders = read("model:orders", admin).await.expect("orders lineage"); - assert_eq!( - orders - .edges - .iter() - .map(|e| (e.source_key.as_str(), e.target_key.as_str())) - .collect::>(), - test_case.expected_orders_edges, - "{}: both directions, never another node's chain or a node outside the snapshot", - test_case.description - ); - assert_eq!( - json!(orders.nodes), - json!([ - {"node_key": "model:order_totals", "kind": "model", "columns": [{"name": "total", "data_type": null}]}, - {"node_key": "model:orders", "kind": "model", "columns": [{"name": "amount", "data_type": "DECIMAL(10,2)"}]}, - {"node_key": "model:raw_orders", "kind": "model", "columns": [{"name": "amount", "data_type": "VARCHAR"}]}, - ]), - "{}", - test_case.description - ); - let leaf = read("model:order_totals", admin) - .await - .expect("leaf lineage"); - assert_eq!( - leaf.edges.len(), - test_case.expected_leaf_edges, - "{}: upstream is followed transitively", - test_case.description - ); - let stranger = create_principal(store, &audit, "orders-stranger") - .await - .expect("person"); - assert!( - matches!( - read("model:orders", stranger).await, - Err(ControlError::Denied { .. }) - ), - "{}: lineage needs view on every node", - test_case.description - ); - close_test_store(fixture).await; - }); - } -} diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 584885c2a0..b59038211b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -38,8 +38,6 @@ mod test_changes; mod test_checks; #[path = "test_cli_login.rs"] mod test_cli_login; -#[path = "test_column_lineage.rs"] -mod test_column_lineage; #[path = "test_contract.rs"] mod test_contract; #[path = "test_custom_roles.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 2ea6a1020a..5d3d2b50f3 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -76,13 +76,6 @@ pub(crate) struct LocationWorkTestCase { pub(crate) expected_generation: i64, } -pub(crate) struct ColumnLineageTestCase { - pub(crate) description: &'static str, - pub(crate) backend: Backend, - pub(crate) expected_orders_edges: Vec<(&'static str, &'static str)>, - pub(crate) expected_leaf_edges: usize, -} - pub(crate) struct WarehouseMetadataTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index 5c4301a8d5..79131a3bb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -2,5 +2,4 @@ pub(crate) mod command; mod constants; mod runtime; mod snapshot; -mod snapshot_lineage; mod worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 6e2353b734..e5e7028962 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -19,7 +19,6 @@ use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project_roles; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::native_host::orchestration_declarations::collect_declarations; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; @@ -80,11 +79,7 @@ pub(crate) fn build_orchestration_snapshot( )?; } input = compile_project_roles(py, input, (&project_dir, graph.bind(py).get().graph()))?; - let project = project.into_pyobject(py)?; - if let Some(lineage) = column_lineage_document(project.as_any())? { - input.documents.insert("lineage".into(), lineage); - } - input.nodes = identity_nodes(project.as_any(), input.nodes)?; + input.nodes = identity_nodes(project.into_pyobject(py)?.as_any(), input.nodes)?; Ok(NativeOrchestrationSnapshot { input }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs deleted file mode 100644 index d6b65e7ef2..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_lineage.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! The project's column lineage edges, published so people can trace columns without the project. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use serde_json::{Value, json}; - -use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; -use crate::bindings::_helpers::records::construct::enum_member; -use crate::bindings::_helpers::records::fields::field; -use crate::bindings::_helpers::records::ids::RecordId; - -/// FAST column lineage of every model as `[source key, column, target key, column, transform]` rows; `None` without SQL analysis. -pub(crate) fn column_lineage_document(project: &Bound<'_, PyAny>) -> PyResult> { - let py = project.py(); - let mode = enum_member(py, RecordId::ColumnLineageMode, "FAST")?; - let Some(lineage) = build_project_column_lineage(project, None, &mode, None)? else { - return Ok(None); - }; - let edges = field(lineage.bind(py), "edges")?; - let mut rows: Vec = Vec::new(); - for edge in edges.try_iter()? { - let edge = edge?; - let (source, target) = (field(&edge, "source")?, field(&edge, "target")?); - rows.push(json!([ - node_key(&source)?, - field(&source, "column_name")?.extract::()?, - node_key(&target)?, - field(&target, "column_name")?.extract::()?, - field(&edge, "transform_kind")?.str()?.extract::()?, - ])); - } - Ok(Some(json!({ "edges": rows }))) -} - -/// The snapshot node key of a lineage column's resource, such as `model:orders`. -fn node_key(column: &Bound<'_, PyAny>) -> PyResult { - let kind: String = field(column, "resource_type")?.str()?.extract()?; - let name: String = field(column, "resource_name")?.extract()?; - Ok(format!("{kind}:{name}")) -} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 8291e2a98c..9fd65cab7a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -233,13 +233,6 @@ class SnapshotEnvironmentTestCase: expected_environment_count: int -@dataclass(frozen=True) -class SnapshotLineageTestCase: - description: str - files: dict[str, str] - expected_edges: list[list[str]] - - @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 9824426203..7437f8ca6a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -35,7 +35,6 @@ SnapshotDiscoveryTestCase, SnapshotEnvironmentTestCase, SnapshotIdentityTestCase, - SnapshotLineageTestCase, SnapshotRejectionTestCase, SnapshotScheduleTestCase, ) @@ -327,59 +326,6 @@ def test_given_invalid_compilation_or_forged_nodes_when_publishing_then_no_snaps ) -@pytest.mark.parametrize( - "test_case", - [ - SnapshotLineageTestCase( - description="published column lineage follows passthroughs, casts and aggregations", - files={ - "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', - "models/stg_orders.sql": ( - "MODEL (description 'Staged orders.');\n\n" - "SELECT 1 AS order_id, 'eu' AS region, '5.00' AS amount_text\n" - ), - "models/orders.sql": ( - "MODEL (description 'Typed orders.');\n\n" - "SELECT order_id, region, CAST(amount_text AS DECIMAL(10, 2)) AS amount\n" - 'FROM __ref("stg_orders")\n' - ), - "models/region_revenue.sql": ( - "MODEL (description 'Revenue by region.');\n\n" - "SELECT region, SUM(amount) AS revenue\n" - 'FROM __ref("orders")\nGROUP BY region\n' - ), - }, - expected_edges=[ - ["model:orders", "amount", "model:region_revenue", "revenue", "aggregation"], - ["model:orders", "region", "model:region_revenue", "region", "direct"], - ["model:stg_orders", "amount_text", "model:orders", "amount", "cast"], - ["model:stg_orders", "order_id", "model:orders", "order_id", "direct"], - ["model:stg_orders", "region", "model:orders", "region", "direct"], - ], - ), - ], - ids=lambda case: case.description, -) -def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_snapshot_document( - test_case: SnapshotLineageTestCase, - tmp_path: Path, -) -> None: - write_files(project_dir=tmp_path, files=test_case.files) - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discover_project_inputs(project_dir=tmp_path), - adapter=DuckDbAdapter(), - resolved_connection={"database": "warehouse", "schema": "main"}, - ) - publication: dict[str, Any] = json.loads( - build_orchestration_snapshot(project, json.dumps(METADATA), tmp_path).to_json() - ) - edges: list[list[str]] = publication["documents"]["lineage"]["edges"] - assert sorted(edges) == test_case.expected_edges - sources: set[str] = {edge[0] for edge in edges} - targets: set[str] = {edge[2] for edge in edges} - assert sources | targets <= set(publication["nodes"]) - - @pytest.mark.parametrize( "test_case", [ diff --git a/ui/src/lib/api/http-catalog.test.ts b/ui/src/lib/api/http-catalog.test.ts index 398299bc76..12323431a7 100644 --- a/ui/src/lib/api/http-catalog.test.ts +++ b/ui/src/lib/api/http-catalog.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { components } from '$control/schema.d.ts'; -import { findNode, nodeDetailOf, toCatalogNodes, toColumnLineage } from './http-catalog'; +import { findNode, nodeDetailOf, toCatalogNodes } from './http-catalog'; import { toCheck } from './http-checks'; type W = components['schemas']; @@ -97,38 +97,3 @@ describe('catalog node detail', () => { expect(withSql.sql).toEqual({ raw: 'select {{ x }}', compiled: 'select {{ x }}' }); }); }); - -describe('column lineage mapping', () => { - const nodes = toCatalogNodes([node('model:raw_orders', { materialization: 'view' }), node('model:orders', { materialization: 'table' })]); - const edge = (source: string, target: string, transform: string): W['ColumnLineageEdge'] => ({ source_key: 'model:raw_orders', source_column: source, target_key: 'model:orders', target_column: target, transform }); - - it('keeps a name through direct or star as a passthrough, a new name as a rename, and names any other transform', () => { - const lineage = toColumnLineage( - { - snapshot_id: 's', - nodes: [ - { node_key: 'model:raw_orders', kind: 'model', columns: [{ name: 'amount', data_type: null }] }, - { node_key: 'model:orders', kind: 'model', columns: [{ name: 'amount', data_type: 'DECIMAL(10,2)' }] }, - { node_key: 'model:retired', kind: 'model', columns: [] } - ], - edges: [edge('id', 'id', 'direct'), edge('note', 'note', 'star'), edge('cust', 'customer_id', 'direct'), edge('amount', 'amount', 'cast')] - }, - nodes - ); - expect(lineage?.edges.map(([, target, lens, label]) => [target, lens, label])).toEqual([ - ['model:orders.id', 'passthrough', undefined], - ['model:orders.note', 'passthrough', undefined], - ['model:orders.customer_id', 'rename', undefined], - ['model:orders.amount', 'transform', 'Cast'] - ]); - expect(lineage?.models.map((m) => [m.id, m.name, m.kind, m.columns])).toEqual([ - ['model:raw_orders', 'raw_orders', 'view', [{ name: 'amount', type: '' }]], - ['model:orders', 'orders', 'table', [{ name: 'amount', type: 'DECIMAL(10,2)' }]] - ]); - expect([lineage?.sql, lineage?.expressions]).toEqual([{}, {}]); - }); - - it('has nothing to show when no lineage reaches the node', () => { - expect(toColumnLineage({ snapshot_id: 's', nodes: [], edges: [] }, nodes)).toBeNull(); - }); -}); diff --git a/ui/src/lib/api/http-catalog.ts b/ui/src/lib/api/http-catalog.ts index 76b50cb991..5c9c492fbd 100644 --- a/ui/src/lib/api/http-catalog.ts +++ b/ui/src/lib/api/http-catalog.ts @@ -1,7 +1,6 @@ import type { components } from '$control/schema.d.ts'; import { shortName } from '$lib/build-area/labels'; -import { cardStatus } from '$lib/lineage/catalog-graph'; -import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, ColumnLens, ColumnLineage, StaleReason } from './types/build'; +import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, StaleReason } from './types/build'; import type { NodeKind } from './types/operate'; type Wire = components['schemas']; @@ -105,34 +104,3 @@ export function nodeDetailOf(nodes: CatalogNode[], node: CatalogNode, status: Wi history: null }; } - -/** A kept name through `direct` or `star` is a passthrough, a new name a rename; every other transform changes the value. */ -function lensOf(edge: Wire['ColumnLineageEdge']): ColumnLens { - if (edge.transform !== 'direct' && edge.transform !== 'star') return 'transform'; - return edge.source_column === edge.target_column ? 'passthrough' : 'rename'; -} - -/** The compiler's transform kind as a label, such as `Cast`; passthroughs and renames keep the lens's own name. */ -function transformLabel(edge: Wire['ColumnLineageEdge']): string | undefined { - if (lensOf(edge) !== 'transform') return undefined; - return edge.transform === 'unknown' ? 'Transformed' : edge.transform.charAt(0).toUpperCase() + edge.transform.slice(1); -} - -/** - * Column lineage around one node, named and coloured from the catalog listing. The read carries no - * SQL or expressions, so those stay empty; null before a snapshot publishes lineage for the node. - */ -export function toColumnLineage(read: Wire['ColumnLineageRead'], nodes: CatalogNode[]): ColumnLineage | null { - if (!read.edges.length) return null; - const listed = new Map(nodes.map((n) => [n.key, n])); - return { - models: read.nodes.flatMap((n) => { - const node = listed.get(n.node_key); - const columns = n.columns.map((c) => ({ name: c.name, type: c.data_type ?? '' })); - return node ? [{ id: n.node_key, name: shortName(n.node_key), kind: node.kind, status: cardStatus(node), columns }] : []; - }), - edges: read.edges.map((e) => [`${e.source_key}.${e.source_column}`, `${e.target_key}.${e.target_column}`, lensOf(e), transformLabel(e)]), - sql: {}, - expressions: {} - }; -} diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 6600d53438..952caa2133 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -2,13 +2,13 @@ import createFetch from 'openapi-fetch'; import type { components, paths } from '$control/schema.d.ts'; import { ApiError, type ApiClient } from './client'; import { checkDetailOf, toCheck, toCheckEvidence, toCheckSql } from './http-checks'; -import { findNode, nodeDetailOf, toCatalogNodes, toColumnLineage } from './http-catalog'; +import { findNode, nodeDetailOf, toCatalogNodes } from './http-catalog'; import { toSnapshotChanges } from './http-changes'; import { jobDetailOf, toJob, toTick, toTrigger, triggerDetailOf } from './http-jobs'; import { RUN_WINDOW, runDetailOf, runRowPage, runsMatching, runTabCountsOf, toRunNode, toRuns } from './http-runs'; import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals'; import type { AuthMode, Bootstrap, Capability, Location, Scope } from './types'; -import type { CatalogNode, CatalogNodeDetail, Check, ColumnLineage, CheckDetail, CheckEvidence, CheckSql, SnapshotChange } from './types/build'; +import type { CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, SnapshotChange } from './types/build'; import type { Job, JobDetail, RerunMode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState } from './types/operate'; type Wire = components['schemas']; @@ -39,7 +39,6 @@ const SUPPORTED: ReadonlySet = new Set([ 'getCheckEvidence', 'listCatalogNodes', 'getCatalogNode', - 'getColumnLineage', 'listSnapshotChanges' ]); @@ -264,18 +263,6 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt if (!snapshots.data) throw new ApiError(errorText(snapshots.error, snapshots.response), snapshots.response.status); return toSnapshotChanges(snapshots.data, runs); }, - async getColumnLineage(scope: Scope, ref: string): Promise { - const nodes = await implemented.listCatalogNodes(scope); - const node = findNode(nodes, ref); - if (!node) return null; - const { locationId, target } = parseLocationKey(scope.locationId); - const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/column-lineage', { - params: { path: { location_id: locationId, target }, query: { node_key: node.key } } - }); - if (response.status === 401) onUnauthorized(); - if (!data) throw new ApiError(errorText(error, response), response.status); - return toColumnLineage(data, nodes); - }, async getCatalogNode(scope: Scope, ref: string): Promise { const nodes = await implemented.listCatalogNodes(scope); const node = findNode(nodes, ref); diff --git a/ui/src/lib/api/types/build.ts b/ui/src/lib/api/types/build.ts index 81e534b63f..906f9e77df 100644 --- a/ui/src/lib/api/types/build.ts +++ b/ui/src/lib/api/types/build.ts @@ -193,9 +193,9 @@ export type CatalogNodeDetail = Omit & { export type ColumnLens = 'raw' | 'passthrough' | 'rename' | 'transform'; export type ColumnLineage = { - models: { id: string; name: string; kind: string; status: 'ok' | 'fail' | 'warn' | 'stale' | 'src' | 'none'; columns: { name: string; type: string }[] }[]; - /** `model.column` pairs from upstream to downstream, with an optional label naming the transform, such as `Cast`. */ - edges: [string, string, ColumnLens, string?][]; + models: { id: string; name: string; kind: string; status: 'ok' | 'fail' | 'warn' | 'stale' | 'src'; columns: { name: string; type: string }[] }[]; + /** `model.column` pairs from upstream to downstream. */ + edges: [string, string, ColumnLens][]; sql: Record; /** The defining expression of each column, per model. */ expressions: Record>; diff --git a/ui/src/lib/components/lineage/col-cell-node.svelte b/ui/src/lib/components/lineage/col-cell-node.svelte index 0ed67190b9..c1f24b1380 100644 --- a/ui/src/lib/components/lineage/col-cell-node.svelte +++ b/ui/src/lib/components/lineage/col-cell-node.svelte @@ -7,10 +7,9 @@ model: string; column: string; kind: string; - status: 'ok' | 'fail' | 'warn' | 'stale' | 'src' | 'none'; + status: 'ok' | 'fail' | 'warn' | 'stale' | 'src'; lens?: Lens; - /** What the incoming edge does, such as `Cast`. */ - label?: string; + lensLabel?: Record; focal?: boolean; active?: boolean; }; @@ -23,8 +22,7 @@ fail: 'var(--sb-error)', warn: 'var(--sb-warning)', stale: 'var(--sb-stale)', - src: 'var(--sb-text-faint)', - none: 'var(--border)' + src: 'var(--sb-text-faint)' }; const KindIcon = $derived(d.kind === 'view' ? Eye : d.kind.startsWith('source') ? Database : d.kind.startsWith('incremental') ? Layers : Table2); const kindColor = $derived(d.kind === 'view' ? '#5aa9e6' : d.kind === 'table' ? '#2e90ff' : d.kind.startsWith('incremental') ? '#3fb6c9' : '#7c8aa5'); @@ -33,22 +31,21 @@
    - {d.column} + {d.column}
    - - {d.model} - {#if d.lens && d.label} - {d.label} + + {d.model} + {#if d.lens && d.lensLabel} + {d.lensLabel[d.lens]} {:else} {d.kind} {/if} diff --git a/ui/src/lib/lineage/column-lineage.svelte b/ui/src/lib/lineage/column-lineage.svelte index 00096734b7..b418a1ab07 100644 --- a/ui/src/lib/lineage/column-lineage.svelte +++ b/ui/src/lib/lineage/column-lineage.svelte @@ -12,7 +12,7 @@ import { highlightSqlbuild } from '$lib/sql/highlight'; import { tick } from 'svelte'; - type ColModel = { id: string; name: string; kind: string; status: 'ok' | 'fail' | 'warn' | 'stale' | 'src' | 'none'; columns: { name: string; type: string }[] }; + type ColModel = { id: string; name: string; kind: string; status: 'ok' | 'fail' | 'warn' | 'stale' | 'src'; columns: { name: string; type: string }[] }; type Lens = 'raw' | 'passthrough' | 'rename' | 'transform'; let { @@ -24,7 +24,7 @@ }: { focal: string; models: ColModel[]; - colEdges: [string, string, Lens, string?][]; + colEdges: [string, string, Lens][]; /** per-model SQL: modelId -> SQL text */ modelSql?: Record; /** per-model column expressions: modelId -> { colName -> defining substring } */ @@ -70,9 +70,6 @@ const kindOf = (id: string) => models.find((m) => m.id === id)?.kind ?? 'table'; const statusOf = (id: string) => models.find((m) => m.id === id)?.status ?? 'ok'; const incomingLens = $derived(new Map(colEdges.map(([, t, lens]) => [t, lens]))); - // What a column's incoming edge does: the published transform, such as Cast, or the lens's name. - const incomingLabel = $derived(new Map(colEdges.map(([, t, lens, label]) => [t, label ?? lensLabel[lens]]))); - const labelOf = (cid: string) => incomingLabel.get(cid) ?? lensLabel.raw; const focalModel = $derived(models.find((m) => m.id === focal)); let colQuery = $state(''); @@ -226,7 +223,7 @@ kind: kindOf(modelOf(cid)), status: statusOf(modelOf(cid)), lens: incomingLens.get(cid), - label: incomingLens.has(cid) ? labelOf(cid) : undefined, + lensLabel, focal: cid === selectedCol, active: cid === activeStep } @@ -287,9 +284,9 @@ onclick={() => (selectedCol = cid)} > {#if selectedCol === cid}{/if} - {c.name} - {lens ? labelOf(cid) : ''} - {#if c.type}{c.type.toLowerCase()}{/if} + {c.name} + {lens ? lensLabel[lens] : ''} + {c.type.toLowerCase()} {:else}
    No columns match.
    @@ -372,8 +369,8 @@
    • @@ -386,8 +383,8 @@
    • @@ -397,7 +394,7 @@ {#if sqlParts && pinnedNode}
    SQL · {sqlParts.model}.sql
    {@html sqlParts.before}{#if sqlParts.match}{@html sqlParts.match}{/if}{@html sqlParts.after}
    - {:else if modelSql && Object.keys(modelSql).length} + {:else}

    Select a step to see it in the model's SQL.

    {/if} @@ -501,14 +498,11 @@ border-color: var(--primary); background: var(--primary); } - /* The identifier being traced wraps rather than truncates. */ .dv-name { + overflow: hidden; font: 12.5px var(--font-mono); - overflow-wrap: anywhere; - } - .col-name { - min-width: 0; - overflow-wrap: anywhere; + text-overflow: ellipsis; + white-space: nowrap; } .dv-name .m { color: var(--muted-foreground);