Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 44 additions & 1 deletion ui/src/lib/api/http-catalog.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { describe, expect, it } from 'vitest';
import type { components } from '$control/schema.d.ts';
import { toCatalogNodes } from './http-catalog';
import { findNode, nodeDetailOf, toCatalogNodes } from './http-catalog';
import { toCheck } from './http-checks';

type W = components['schemas'];

Expand Down Expand Up @@ -53,4 +54,46 @@ describe('catalog node mapping', () => {
expect([orders.description, orders.tags, orders.columnCount, orders.relation]).toEqual(['One row per order', ['marts'], 4, { database: null, schema: 'main', name: 'orders' }]);
expect([orders.stale, orders.checks, orders.lastOutcome, orders.group, orders.owner]).toEqual([null, null, null, null, []]);
});

it('finds a node by key, or by bare name preferring the model', () => {
const nodes = toCatalogNodes([node('source:orders'), node('source:customers'), node('model:orders', { materialization: 'view' })]);
expect([findNode(nodes, 'source:orders')?.key, findNode(nodes, 'orders')?.key, findNode(nodes, 'customers')?.key, findNode(nodes, 'order')]).toEqual(['source:orders', 'model:orders', 'source:customers', null]);
});
});

function check(id: string, target: string, outcome: string): W['CheckResult'] {
return { event_id: `e-${id}`, binding_key: id, audit_name: id, audit_definition_name: 'not_null', attachment_kind: 'model', attached_target_name: target, severity: 'error', evaluation_mode: 'violations', run_scope_phase: 'final', occurred_at: '2026-11-07T06:00:00Z', run_id: 'r', invocation_id: 'i', outcome, reused: 'false' };
}

function status(definition: W['NodeStatus']['definition'], physical: W['NodeStatus']['physical']): W['NodeStatus'] {
const build = { definition_hash: 'd', event_id: 'e', observed_at: '2026-11-07T06:00:00Z', run_id: 'sqb-run-1', schema_fingerprint: 's', source_event_id: 'x', version_hash: 'v1' };
return { node_key: 'model:orders', snapshot_id: 'snap', build, definition, physical, source_data: { status: 'unknown', reason: 'not tracked' } };
}

describe('catalog node detail', () => {
const nodes = toCatalogNodes([node('model:orders', { materialization: 'table' }), node('model:order_totals', { materialization: 'view', dependencies: ['model:orders'] }), node('model:orders_by_day', { materialization: 'view' })]);
const checks = [check('a', 'orders', 'pass'), check('b', 'model:orders', 'fail'), check('c', 'orders', 'warn'), check('d', 'orders_by_day', 'fail'), { ...check('e', 'orders', 'fail'), attachment_kind: 'source' }].map(toCheck);
const observed: W['NodeStatus']['physical'] = { status: 'observed', observation: { event_id: 'o', observed_at: '2026-11-07T06:00:00Z', record: { name: 'orders', relation_type: 'table', state_schema: 'main', columns: [{ name: 'order_id', data_type: 'INTEGER' }] } } };

it('keeps only its own checks, matched by key or by bare name of the same kind, and counts them by newest outcome', () => {
const d = nodeDetailOf(nodes, nodes[0], null, null, checks);
expect(d.checkResults.map((c) => c.id)).toEqual(['a', 'b', 'c']);
expect(d.checks).toEqual({ total: 3, passing: 1, warning: 1, failing: 1 });
expect(d.downstream).toEqual(['model:order_totals']);
});

it('reports a changed definition as stale and maps the warehouse columns', () => {
const changed = nodeDetailOf(nodes, nodes[0], status({ status: 'changed', current_local_hash: 'b', recorded_local_hash: 'a' }, observed), null, []);
expect(changed.stale.map((r) => [r.reason, r.since])).toEqual([['changed', '2026-11-07T06:00:00Z']]);
expect(changed.columns).toEqual([{ name: 'order_id', type: 'INTEGER', description: '', audits: [] }]);
const matched = nodeDetailOf(nodes, nodes[0], status({ status: 'matched', local_hash: 'a' }, { status: 'unavailable', reason: 'no catalog' }), null, []);
expect([matched.stale, matched.columns]).toEqual([[], []]);
});

it('leaves refused SQL out and unreported history unknown rather than empty', () => {
const refused = nodeDetailOf(nodes, nodes[0], null, null, []);
expect([refused.sql, refused.runs, refused.jobs, refused.history]).toEqual([null, null, null, null]);
const withSql = nodeDetailOf(nodes, nodes[0], null, { node_key: 'model:orders', snapshot_id: 'snap', raw_sql: 'select {{ x }}', compiled_sql: null }, []);
expect(withSql.sql).toEqual({ raw: 'select {{ x }}', compiled: 'select {{ x }}' });
});
});
58 changes: 57 additions & 1 deletion ui/src/lib/api/http-catalog.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { components } from '$control/schema.d.ts';
import type { CatalogNode } from './types/build';
import { shortName } from '$lib/build-area/labels';
import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, StaleReason } from './types/build';
import type { NodeKind } from './types/operate';

type Wire = components['schemas'];
Expand Down Expand Up @@ -48,3 +49,58 @@ export function toCatalogNodes(wire: Wire['CatalogNodeRecord'][]): CatalogNode[]
const listed = new Set(nodes.map((node) => node.key));
return nodes.map((node) => ({ ...node, upstream: node.upstream.filter((key) => listed.has(key)) }));
}

/** Passing, warning and failing by each check's newest result; checks without one count only towards the total. */
export function checkCountsOf(checks: Check[]): CheckCounts {
const outcome = (c: Check) => c.latest?.outcome;
return {
total: checks.length,
passing: checks.filter((c) => outcome(c) === 'pass').length,
warning: checks.filter((c) => outcome(c) === 'warn').length,
failing: checks.filter((c) => ['fail', 'error'].includes(outcome(c) ?? '')).length
};
}

/** A definition that changed since its last recorded build here; other staleness is not reported by the server. */
function staleOf(status: Wire['NodeStatus'] | null): StaleReason[] {
if (status?.definition.status !== 'changed' || !status.build) return [];
return [{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: status.build.observed_at }];
}


/** The listed node a key or bare name refers to, preferring a model; migrations name models without their kind. */
export function findNode(nodes: CatalogNode[], ref: string): CatalogNode | null {
return nodes.find((n) => n.key === ref) ?? nodes.find((n) => n.key === `model:${ref}`) ?? nodes.find((n) => shortName(n.key) === ref) ?? null;
}

/**
* A listed node with its status and SQL reads and its checks. The server keeps no
* per-node run history, row counts, cost, jobs or migrations, so those stay empty or null and the
* pages hide them. `sql` is null when the person may not see SQL.
*/
export function nodeDetailOf(nodes: CatalogNode[], node: CatalogNode, status: Wire['NodeStatus'] | null, sql: Wire['NodeSql'] | null, checks: Check[]): CatalogNodeDetail {
// Checks name their node without its kind, so the attachment tells a source from a model of the same name.
const own = checks.filter((c) => c.node === node.key || `${c.attachment}:${c.node}` === node.key);
const physical = status?.physical.status === 'observed' ? status.physical.observation.record : null;
return {
...node,
stale: staleOf(status),
checks: checkCountsOf(own),
downstream: nodes.filter((n) => n.upstream.includes(node.key)).map((n) => n.key),
recentOutcomes: [],
lastRows: null,
rowsDelta: null,
lastDurationMs: null,
cost: null,
columns: (physical?.columns ?? []).map((c) => ({ name: c.name, type: c.data_type, description: '', audits: [] })),
sql: sql?.raw_sql ? { raw: sql.raw_sql, compiled: sql.compiled_sql ?? sql.raw_sql } : null,
code: null,
builds: [],
jobs: null,
runs: null,
checkResults: own,
warehouse: null,
microbatch: null,
history: null
};
}
47 changes: 46 additions & 1 deletion ui/src/lib/api/http-client.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { HttpChangeFeed, feedAllowed } from './http-client';
import { HttpChangeFeed, createHttpClient, feedAllowed } from './http-client';
import type { ChangedSignal, ConnectionState } from './signals';

/** A stand-in `EventSource` the test opens, fails and feeds by hand. */
Expand Down Expand Up @@ -237,3 +237,48 @@ describe('feedAllowed', () => {
await expect(feedAllowed('', ORDERS, () => {})).rejects.toThrow('network down');
});
});

describe('getCatalogNode', () => {
/** Answers the reads a node page makes; anything else, such as node status, is refused. */
function serve(actions: string[], sqlStatus = 200) {
const paths: string[] = [];
const answers: [string, unknown][] = [
['/bootstrap', { login_mode: 'password', access: { name: 'vic', principal_id: 'p', actions: [], locations: [{ location_id: '0191f0aa-orders', name: 'Orders', target: 'development', actions }] } }],
['/nodes', [{ node_key: 'model:orders', kind: 'model', name: 'orders', materialization: 'table', tags: [], dependencies: [], snapshot_id: 's' }]],
['/checks', []],
['/sql', { node_key: 'model:orders', snapshot_id: 's', raw_sql: 'select 1', compiled_sql: 'select 1' }]
];
vi.stubGlobal(
'fetch',
vi.fn(async (request: Request) => {
const path = new URL(request.url).pathname;
paths.push(path);
const answer = answers.find(([end]) => path.endsWith(end));
const refused = !answer || (answer[0] === '/sql' && sqlStatus === 403);
return refused ? Response.json({ error: 'refused' }, { status: 403 }) : Response.json(answer[1]);
})
);
return paths;
}

it.each([
[['view', 'view.sql'], 'select 1'],
[['view'], null]
])('with %j reads SQL only when the location allows it, so a viewer causes no refused read', async (actions, raw) => {
const paths = serve(actions);
const client = createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} });
await client.bootstrap();
const detail = await client.getCatalogNode(ORDERS, 'orders');
expect([detail?.key, detail?.sql?.raw ?? null]).toEqual(['model:orders', raw]);
expect(paths.filter((p) => p.endsWith('/sql'))).toHaveLength(raw ? 1 : 0);
});

it('asks for a refused node SQL once, when view.sql holds on only some nodes', async () => {
const paths = serve(['view', 'view.sql'], 403);
const client = createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} });
await client.bootstrap();
const loads = [await client.getCatalogNode(ORDERS, 'orders'), await client.getCatalogNode(ORDERS, 'orders')];
expect(loads.map((d) => [d?.key, d?.sql])).toEqual([['model:orders', null], ['model:orders', null]]);
expect(paths.filter((p) => p.endsWith('/sql'))).toHaveLength(1);
});
});
31 changes: 28 additions & 3 deletions ui/src/lib/api/http-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@ import createFetch from 'openapi-fetch';
import type { components, paths } from '$control/schema.d.ts';
import { ApiError, type ApiClient } from './client';
import { checkDetailOf, toCheck, toCheckEvidence, toCheckSql } from './http-checks';
import { toCatalogNodes } from './http-catalog';
import { findNode, nodeDetailOf, toCatalogNodes } from './http-catalog';
import { toSnapshotChanges } from './http-changes';
import { jobDetailOf, toJob, toTick, toTrigger, triggerDetailOf } from './http-jobs';
import { RUN_WINDOW, runDetailOf, runRowPage, runsMatching, runTabCountsOf, toRunNode, toRuns } from './http-runs';
import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals';
import type { AuthMode, Bootstrap, Capability, Location, Scope } from './types';
import type { CatalogNode, Check, CheckDetail, CheckEvidence, CheckSql, SnapshotChange } from './types/build';
import type { CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, SnapshotChange } from './types/build';
import type { Job, JobDetail, RerunMode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState } from './types/operate';

type Wire = components['schemas'];
Expand Down Expand Up @@ -38,6 +38,7 @@ const SUPPORTED: ReadonlySet<string> = new Set<keyof ApiClient>([
'getCheckSql',
'getCheckEvidence',
'listCatalogNodes',
'getCatalogNode',
'listSnapshotChanges'
]);

Expand Down Expand Up @@ -186,14 +187,20 @@ export type HttpClientOptions = {
*/
export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOptions): ApiClient {
const http = createFetch<paths>({ baseUrl, credentials: 'same-origin' });
// Locations whose bootstrap actions include view.sql, which may hold on only some nodes, and the
// nodes there whose SQL was refused; each refused read is audited, so neither is asked again until reload.
let sqlLocations = new Set<string>();
const sqlRefused = new Set<string>();
const implemented = {
supports: (method: keyof ApiClient) => SUPPORTED.has(method),
changes: new HttpChangeFeed(baseUrl, (scope) => feedAllowed(baseUrl, scope, onUnauthorized)),
async bootstrap(): Promise<Bootstrap> {
const { data, error, response } = await http.GET('/api/control/bootstrap');
if (response.status === 401) onUnauthorized();
if (!data) throw new ApiError(errorText(error, response), response.status);
return toBootstrap(data);
const bootstrap = toBootstrap(data);
sqlLocations = new Set(bootstrap.locations.filter((l) => l.capabilities?.includes('view.sql')).map((l) => l.id));
return bootstrap;
},
listRunRows: async (scope: Scope, query: RunRowQuery = {}): Promise<RunRowPage> => runRowPage(await listedRuns(scope), scope.locationId, query),
runTabCounts: async (scope: Scope, query: RunRowQuery = {}): Promise<RunTabCounts> => runTabCountsOf(await listedRuns(scope), scope.locationId, query),
Expand Down Expand Up @@ -256,6 +263,24 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt
if (!snapshots.data) throw new ApiError(errorText(snapshots.error, snapshots.response), snapshots.response.status);
return toSnapshotChanges(snapshots.data, runs);
},
async getCatalogNode(scope: Scope, ref: string): Promise<CatalogNodeDetail | null> {
const nodes = await implemented.listCatalogNodes(scope);
const node = findNode(nodes, ref);
if (!node) return null;
const { locationId, target } = parseLocationKey(scope.locationId);
const path = { location_id: locationId, target };
const [status, sql, checks] = await Promise.all([
http.GET('/api/control/locations/{location_id}/targets/{target}/node-status', { params: { path, query: { node_key: node.key } } }),
sqlLocations.has(scope.locationId) && !sqlRefused.has(`${scope.locationId}|${node.key}`)
? http.GET('/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql', { params: { path: { ...path, node_key: node.key } } })
: null,
checkResults(scope).catch(() => [])
]);
if (status.response.status === 401 || sql?.response.status === 401) onUnauthorized();
if (sql?.response.status === 403) sqlRefused.add(`${scope.locationId}|${node.key}`);
// Status and SQL have their own permissions; a refusal leaves that part out rather than failing the page.
return nodeDetailOf(nodes, node, status.data ?? null, sql?.data ?? null, checks.map(toCheck));
},
listChecks: async (scope: Scope): Promise<Check[]> => (await checkResults(scope)).map(toCheck),
getCheck: async (scope: Scope, id: string): Promise<CheckDetail | null> => checkDetailOf(await checkResults(scope, id)),
async getCheckSql(scope: Scope, checkId: string): Promise<CheckSql> {
Expand Down
9 changes: 6 additions & 3 deletions ui/src/lib/api/types/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -177,12 +177,15 @@ export type CatalogNodeDetail = Omit<CatalogNode, 'stale' | 'checks'> & {
code: { file: string; source: string } | null;
/** As built in this target, newest first. */
builds: BuiltDefinition[];
jobs: NodeJob[];
runs: NodeRunEntry[];
/** Null when the server does not report which jobs select the node. */
jobs: NodeJob[] | null;
/** Null when the server does not report the node's run history. */
runs: NodeRunEntry[] | null;
checkResults: Check[];
warehouse: WarehouseObject | null;
microbatch: MicrobatchCoverage | null;
history: NodeHistory;
/** Null when the server does not report renames and moves. */
history: NodeHistory | null;
};

// ---- column lineage ----
Expand Down
3 changes: 3 additions & 0 deletions ui/src/lib/build-area/labels.ts
Original file line number Diff line number Diff line change
Expand Up @@ -174,3 +174,6 @@ export function relationName(relation: { database: string | null; schema: string
if (!relation) return '—';
return [relation.database, relation.schema, relation.name].filter(Boolean).join('.');
}

/** A node key without its kind, e.g. `customer_summary` for `model:customer_summary`. */
export const shortName = (key: string) => key.slice(key.indexOf(':') + 1);
7 changes: 4 additions & 3 deletions ui/src/lib/components/code/code-preview.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -114,15 +114,14 @@
.ml-auto {
margin-left: auto;
}
/* A preview is read at a glance, so long lines wrap; the full viewer scrolls. */
.body {
padding: 8px 0;
overflow-x: auto;
color: var(--sb-mono);
font: 12px/1.7 var(--font-mono);
}
.line {
display: flex;
min-width: max-content;
padding-right: 12px;
}
.ln {
Expand All @@ -134,8 +133,10 @@
user-select: none;
}
.line code {
min-width: 0;
font: inherit;
white-space: pre;
white-space: pre-wrap;
overflow-wrap: anywhere;
}
.more {
display: block;
Expand Down
4 changes: 3 additions & 1 deletion ui/src/lib/components/lineage/asset-node.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@

type AssetData = {
name: string;
/** Shown instead of the key when set; the key stays in the tooltip. */
label?: string;
kind: string;
/** The last build outcome; staleness is shown separately. */
status: 'ok' | 'fail' | 'warn' | 'run' | 'src' | 'none';
Expand Down Expand Up @@ -98,7 +100,7 @@

<div class="head">
<Glyph size={14} strokeWidth={1.75} class="glyph" />
<span class="name" title={d.name}>{d.name}</span>
<span class="name" title={d.name}>{d.label ?? d.name}</span>
{#if f.kind}<span class="kind">{kindShort}</span>{/if}
</div>

Expand Down
3 changes: 2 additions & 1 deletion ui/src/lib/lineage/catalog-graph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import type { Edge, Node } from '@xyflow/svelte';
import type { CatalogNode, StaleReason } from '$lib/api';
import { kindLine, nodeIconKey } from '$lib/build-area/labels';
import { kindLine, nodeIconKey, shortName } from '$lib/build-area/labels';

export type CardStatus = 'ok' | 'fail' | 'warn' | 'src' | 'none';
export type CardStale = 'changed' | 'compiled' | 'upstream' | 'behind';
Expand Down Expand Up @@ -35,6 +35,7 @@ export function checksBadge(node: CatalogNode): { checks?: string; checksColor?:
export function cardData(node: CatalogNode): Record<string, unknown> {
return {
name: node.key,
label: shortName(node.key),
kind: kindLine(node),
group: node.group,
nodeKind: node.kind,
Expand Down
6 changes: 3 additions & 3 deletions ui/src/lib/lineage/scoped-lineage.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -181,12 +181,12 @@

<div class="ml-auto flex items-center gap-2.5">
<span class="text-[var(--sb-text-faint)] text-[12px]">--select <span class="text-muted-foreground">{selectExpr}</span></span>
<button
{#if onExecute}<button
class="bg-primary rounded-[2px] px-3 py-1 text-[12px] font-medium text-white disabled:opacity-50"
disabled={!app.can('run.adhoc')}
title={app.can('run.adhoc') ? undefined : 'Requires run.adhoc'}
onclick={() => onExecute?.(effectiveSelection.map((id) => graphNodes.find((n) => n.id === id)?.name ?? id))}>▷ Execute {effectiveSelection.length > 1 ? `(${effectiveSelection.length})` : ''}</button
>
onclick={() => onExecute(effectiveSelection.map((id) => graphNodes.find((n) => n.id === id)?.name ?? id))}>▷ Execute {effectiveSelection.length > 1 ? `(${effectiveSelection.length})` : ''}</button
>{/if}
</div>
</div>

Expand Down
5 changes: 2 additions & 3 deletions ui/src/routes/catalog/[name]/history-tab.svelte
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<script lang="ts">
import type { CatalogNodeDetail } from '$lib/api';
import type { CatalogNodeDetail, NodeHistory } from '$lib/api';
import Panel from '$lib/components/platform/panel.svelte';
import StatusLabel from '$lib/components/list/status-label.svelte';
import { app } from '$lib/state/app.svelte';
Expand All @@ -9,8 +9,7 @@

// Renames and moves of this node, from synced `_sqlbuild_migrations` and
// `_sqlbuild_old_name_views`; the Maintenance page shows them for every node.
let { node }: { node: CatalogNodeDetail } = $props();
const h = $derived(node.history);
let { node, history: h }: { node: CatalogNodeDetail; history: NodeHistory } = $props();
</script>

<div class="wrap">
Expand Down
Loading
Loading