feat(ui): job lineage graph, reworked run page and lineage inspector fixes

ui/src/lib/api/client.ts

@@ -64,7 +64,8 @@ export interface ApiClient extends PlatformApi {
 	overview(scope: Scope): Promise<Overview | null>;
 	/** Every node in the current snapshot with its build state; also the lineage graph. */
 	listCatalogNodes(scope: Scope): Promise<CatalogNode[]>;
-	getCatalogNode(scope: Scope, key: string): Promise<CatalogNodeDetail | null>;
+	/** `listing` is a current `listCatalogNodes` result the caller already holds, so it isn't read again. */
+	getCatalogNode(scope: Scope, key: string, listing?: CatalogNode[]): Promise<CatalogNodeDetail | null>;
 	/** The snapshot's column lineage around one node. */
 	getColumnLineage(scope: Scope, key: string): Promise<ColumnLineage | null>;
 	/** Every audit and check with its latest result. */

ui/src/lib/api/http-client.ts

@@ -12,7 +12,7 @@ import { overviewOf } from './http-overview';
 import { toPlatformEvents, toPlatformHealth } from './http-problems';
 import { roleDetailOf, toRoleDefinition } from './http-roles';
 import { jobDetailOf, runWindowOf, toJob, toTick, toTrigger, triggerDetailOf } from './http-jobs';
-import { RUN_WINDOW, toAdhocCheck, runDetailOf, runRowPage, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs';
+import { RUN_WINDOW, groupRunNodes, toAdhocCheck, runDetailOf, runRowPage, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs';
 import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals';
 import type { AuthMode, Bootstrap, Capability, Location, PlatformEvent, Scope, WarehouseSync } from './types';
 import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, LocationSummary, PlatformHealth, MaintenanceHistory, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform';
@@ -259,8 +259,8 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt
 			if (!found) return null;
 			const { run, wire, listed } = found;
 			const owners = [run, ...listed.filter((r) => r.parentRunId === run.id)];
-			const nodes = (await Promise.all(owners.map(async (owner) => (await runNodes(scope, owner.id)).map((n) => toRunNode(n, owner))))).flat();
-			return runDetailOf(run, wire, listed, nodes);
+			const [own, ...children] = await Promise.all(owners.map(async (owner) => (await runNodes(scope, owner.id)).map((n) => toRunNode(n, owner))));
+			return runDetailOf(run, wire, listed, groupRunNodes(own, children.flat()));
 		},
 		async listRunEvents(scope: Scope, runId: string, query: { after?: number; limit?: number } = {}): Promise<RunEventPage> {
 			const limit = query.limit ?? 1000;
@@ -502,8 +502,8 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt
 			if (!data) throw new ApiError(errorText(error, response), response.status);
 			return toColumnLineage(data, nodes);
 		},
-		async getCatalogNode(scope: Scope, ref: string): Promise<CatalogNodeDetail | null> {
-			const nodes = await implemented.listCatalogNodes(scope);
+		async getCatalogNode(scope: Scope, ref: string, listing?: CatalogNode[]): Promise<CatalogNodeDetail | null> {
+			const nodes = listing ?? (await implemented.listCatalogNodes(scope));
 			const node = findNode(nodes, ref);
 			if (!node) return null;
 			const { locationId, target } = parseLocationKey(scope.locationId);

ui/src/lib/api/http-runs.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from 'vitest';
 import type { components } from '$control/schema.d.ts';
-import { runDetailOf, runRowPage, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunStatements, toRunOutput } from './http-runs';
+import { groupRunNodes, runDetailOf, runRowPage, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunStatements, toRunOutput } from './http-runs';
 
 type RunRecord = components['schemas']['RunRecord'];
 type W = components['schemas'];
@@ -176,3 +176,16 @@ describe('run Recall reads', () => {
 	});
 });
 
+
+describe('group run nodes', () => {
+	it("keeps each job run's node and the group's own row only where no job run has the node", () => {
+		const group = toRuns([wire('g', { kind: 'group', job_key: 'nightly' })], LOCATION)[0];
+		const staging = toRuns([wire('s', { parent_run_id: 'g', job_key: 'staging' })], LOCATION)[0];
+		const node = (key: string, status: string, owner = group) => toRunNode({ node_key: key, status } as W['RunNodeRecord'], owner);
+		const nodes = groupRunNodes([node('model:raw', 'success'), node('model:report', 'pending')], [node('model:raw', 'failed', staging)]);
+		expect(nodes.map((n) => [n.nodeKey, n.runId, n.status])).toEqual([
+			['model:report', 'g', 'pending'],
+			['model:raw', 's', 'failed']
+		]);
+	});
+});

ui/src/lib/api/http-runs.ts

@@ -226,6 +226,15 @@ export function runDetailOf(run: Run, wire: Wire['RunRecord'], listed: Run[], no
 	};
 }
 
+/**
+ * A run's nodes with its job runs' nodes. A group run's own rows repeat its job runs' nodes, so they
+ * stay only for nodes no job run has, such as those of members that never started.
+ */
+export function groupRunNodes(own: RunNode[], children: RunNode[]): RunNode[] {
+	const covered = new Set(children.map((n) => n.nodeKey));
+	return [...own.filter((n) => !covered.has(n.nodeKey)), ...children];
+}
+
 const KNOWN_EVENTS: ReadonlySet<string> = new Set(RECALL_EVENT_TYPES);
 /** Scheduler facts are stored as `run_<fact>` events in the scheduler invocation. */
 const SCHEDULER_EVENTS: ReadonlySet<string> = new Set(SCHEDULER_FACTS.map((f) => `run_${f}`));

ui/src/lib/api/mock/build.ts

@@ -100,7 +100,7 @@ export class MockBuild {
 		return this.#ctx.reply(SNAPSHOT_NODES.map((sn) => this.#catalogNode(states.get(sn.key)!, checks)));
 	}
 
-	async getCatalogNode(scope: Scope, key: string): Promise<CatalogNodeDetail | null> {
+	async getCatalogNode(scope: Scope, key: string, _listing?: CatalogNode[]): Promise<CatalogNodeDetail | null> {
 		const states = this.#states(scope);
 		const state = states.get(key);
 		if (!state) return this.#ctx.reply(null);

ui/src/lib/api/mock/operate-store.ts

@@ -296,8 +296,8 @@ export class MockOperate {
 	listRunEvents(scope: Scope, runId: string, query: { after?: number; limit?: number } = {}): Promise<RunEventPage> {
 		const rec = this.#require(runId);
 		const now = Date.now();
-		const all = [rec, ...this.#children(rec)]
-			.flatMap((r) => execution(r, this.#nodesOf(r, now)).events)
+		// Each run's own, as the server returns; the run page gathers a group's job runs itself.
+		const all = execution(rec, this.#nodesOf(rec, now)).events
 			.sort((a, b) => a.occurredAt.localeCompare(b.occurredAt) || a.storageId - b.storageId)
 			.map((e, i) => ({ ...e, storageId: i + 1 }));
 		const after = query.after ?? 0;
@@ -310,24 +310,22 @@ export class MockOperate {
 	listRunStatements(scope: Scope, runId: string): Promise<RunStatement[]> {
 		const rec = this.#require(runId);
 		const now = Date.now();
-		return this.#ctx.reply([rec, ...this.#children(rec)].flatMap((r) => execution(r, this.#nodesOf(r, now), now).statements));
+		return this.#ctx.reply(execution(rec, this.#nodesOf(rec, now), now).statements);
 	}
 
 	getStatementSql(scope: Scope, runId: string, statementId: string): Promise<StatementSql> {
 		this.#need('view.sql');
 		const rec = this.#require(runId);
 		const now = Date.now();
-		const statement = [rec, ...this.#children(rec)]
-			.flatMap((r) => execution(r, this.#nodesOf(r, now), now).statements)
-			.find((st) => st.id === statementId);
+		const statement = execution(rec, this.#nodesOf(rec, now), now).statements.find((st) => st.id === statementId);
 		if (!statement) throw new ApiError(`Unknown statement ${statementId}`, 404);
 		return this.#ctx.reply({ statementId, sql: statement.sqlRecorded ? sqlFor(statement.nodeKey, statement.kind, this.#ctx.target(scope)) : null });
 	}
 
 	listRunOutput(scope: Scope, runId: string): Promise<RunOutputLine[]> {
 		const rec = this.#require(runId);
 		const now = Date.now();
-		return this.#ctx.reply([rec, ...this.#children(rec)].flatMap((r) => execution(r, this.#nodesOf(r, now)).output));
+		return this.#ctx.reply(execution(rec, this.#nodesOf(rec, now)).output);
 	}
 
 	listQueue(scope: Scope): Promise<QueueEntry[]> {

ui/src/lib/components/app-topbar.svelte

@@ -78,5 +78,8 @@
 	.app-topbar {
 		background: color-mix(in srgb, var(--background) 94%, transparent);
 		backdrop-filter: blur(12px);
+		/* The blur makes the bar its own layer; this keeps it and its popovers, such as Run now, above the page. */
+		position: relative;
+		z-index: 20;
 	}
 </style>

ui/src/lib/lineage/job-graph.test.ts

@@ -0,0 +1,58 @@
+import { describe, expect, it } from 'vitest';
+import type { Node } from '@xyflow/svelte';
+import type { CatalogNode } from '$lib/api';
+import { boxId, jobGraph, layoutJobGraph } from './job-graph';
+
+const node = (key: string, upstream: string[] = []) => ({ key, kind: 'table', upstream, owner: [], tags: [], stale: null, checks: null }) as unknown as CatalogNode;
+const measured = (nodes: Node[]) => nodes.map((n) => (n.type === 'job' ? n : { ...n, measured: { width: 236, height: 61 } }));
+
+// staging builds raw → stg; orders reads stg; reports waits for orders only through `after`.
+const catalog = [node('model:raw'), node('model:stg', ['model:raw']), node('model:orders', ['model:stg']), node('model:report')];
+const all = catalog.map((n) => n.key);
+const boxes = [
+	{ jobKey: 'staging', nodes: ['model:raw', 'model:stg'] },
+	{ jobKey: 'orders', nodes: ['model:orders', 'model:stg'] },
+	{ jobKey: 'reports', nodes: ['model:report'] }
+];
+const members = [
+	{ jobKey: 'staging', upstream: [], stage: 0 },
+	{ jobKey: 'orders', upstream: [{ jobKey: 'staging', source: 'lineage' as const }], stage: 1 },
+	{ jobKey: 'reports', upstream: [{ jobKey: 'orders', source: 'after' as const }, { jobKey: 'staging', source: 'lineage' as const }], stage: 2 }
+];
+
+describe('job graph', () => {
+	it('puts each node in the first box that builds it and draws `after` only where no lineage links two boxes', () => {
+		const { nodes, edges } = jobGraph(all, catalog, boxes, members);
+		const parent = Object.fromEntries(nodes.filter((n) => n.type === 'asset').map((n) => [n.id, n.parentId]));
+		expect(parent).toEqual({ 'model:raw': boxId('staging'), 'model:stg': boxId('staging'), 'model:orders': boxId('orders'), 'model:report': boxId('reports') });
+		// Boxes come before their cards, as Svelte Flow requires.
+		expect(nodes.findIndex((n) => n.id === boxId('reports'))).toBeLessThan(nodes.findIndex((n) => n.id === 'model:raw'));
+		expect(edges.map((e) => `${e.source}>${e.target}`).sort()).toEqual(
+			['model:raw>model:stg', 'model:stg>model:orders', `${boxId('orders')}>${boxId('reports')}`, `${boxId('staging')}>${boxId('reports')}`].sort()
+		);
+	});
+
+	it('sizes boxes around their cards, places cards inside them, and orders boxes by their dependencies', () => {
+		const { nodes, edges } = jobGraph(all, catalog, boxes, members);
+		const laid = layoutJobGraph(measured(nodes), edges);
+		const box = (k: string) => laid.find((n) => n.id === boxId(k))!;
+		for (const card of laid.filter((n) => n.parentId)) {
+			const b = laid.find((n) => n.id === card.parentId)!;
+			expect(card.position.x).toBeGreaterThan(0);
+			expect(card.position.y).toBeGreaterThan(0);
+			expect(card.position.x + 236).toBeLessThan(b.width!);
+			expect(card.position.y + 61).toBeLessThan(b.height!);
+		}
+		expect(box('staging').width).toBeGreaterThan(2 * 236);
+		expect(box('staging').position.x + box('staging').width!).toBeLessThan(box('orders').position.x);
+		expect(box('orders').position.x + box('orders').width!).toBeLessThan(box('reports').position.x);
+	});
+
+	it('draws a plain job without boxes', () => {
+		const { nodes } = jobGraph(['model:raw', 'model:stg'], catalog);
+		expect(nodes.map((n) => [n.id, n.type, n.parentId])).toEqual([
+			['model:raw', 'asset', undefined],
+			['model:stg', 'asset', undefined]
+		]);
+	});
+});

ui/src/lib/lineage/job-graph.ts

@@ -0,0 +1,126 @@
+// A job's nodes as a lineage graph; for a group, each member job is a box around the nodes it builds.
+
+import dagre from '@dagrejs/dagre';
+import type { Edge, Node } from '@xyflow/svelte';
+import type { CatalogNode, JobGroupMember, RunRef } from '$lib/api';
+import { shortName } from '$lib/build-area/labels';
+import { cardData } from './catalog-graph';
+
+const NODE_WIDTH = 236;
+const NODE_HEIGHT = 61;
+/** Room inside a member box: padding around its nodes plus its header. */
+const PAD = 14;
+const HEADER = 30;
+const EMPTY_HEIGHT = 44;
+
+export type JobBox = { jobKey: string; nodes: string[]; lastRun?: RunRef | null };
+
+const edgeStyle = 'stroke: var(--sb-edge); stroke-width: 1.5;';
+const afterStyle = 'stroke: var(--sb-edge); stroke-width: 1.5; stroke-dasharray: 5 4;';
+
+export function boxId(jobKey: string): string {
+	return `job:${jobKey}`;
+}
+
+/**
+ * Cards for `nodes` and the lineage between them. With `boxes`, each node sits in the first box
+ * that builds it, and a declared `after` between two members with no lineage between them is a
+ * dashed edge between their boxes.
+ */
+export function jobGraph(nodes: string[], catalog: CatalogNode[], boxes: JobBox[] = [], members: JobGroupMember[] = []): { nodes: Node[]; edges: Edge[] } {
+	const byKey = new Map(catalog.map((n) => [n.key, n]));
+	const inJob = new Set(nodes);
+	const owner = new Map<string, string>();
+	for (const box of boxes) for (const n of box.nodes) if (inJob.has(n) && !owner.has(n)) owner.set(n, box.jobKey);
+
+	const boxNodes: Node[] = boxes.map((b) => ({
+		id: boxId(b.jobKey),
+		type: 'job',
+		position: { x: 0, y: 0 },
+		width: NODE_WIDTH + 2 * PAD,
+		height: EMPTY_HEIGHT,
+		selectable: false,
+		data: { jobKey: b.jobKey, lastRun: b.lastRun ?? null, nodeCount: nodes.filter((n) => owner.get(n) === b.jobKey).length }
+	}));
+	const cards: Node[] = nodes.map((key) => {
+		const parent = owner.get(key);
+		const node = byKey.get(key);
+		return {
+			id: key,
+			type: 'asset',
+			position: { x: 0, y: 0 },
+			...(parent ? { parentId: boxId(parent), extent: 'parent' as const } : {}),
+			data: node ? cardData(node) : { name: key, label: shortName(key), kind: '', status: 'none' }
+		};
+	});
+
+	const lineage: Edge[] = nodes.flatMap((key) =>
+		(byKey.get(key)?.upstream ?? []).filter((up) => inJob.has(up)).map((up) => ({ id: `${up}-${key}`, source: up, target: key, type: 'bezier', style: edgeStyle }))
+	);
+	const linked = new Set(lineage.map((e) => `${owner.get(e.source)}>${owner.get(e.target)}`));
+	const after: Edge[] = members.flatMap((m) =>
+		m.upstream
+			.filter((u) => !linked.has(`${u.jobKey}>${m.jobKey}`) && boxes.some((b) => b.jobKey === u.jobKey))
+			.map((u) => ({ id: `after:${u.jobKey}-${m.jobKey}`, source: boxId(u.jobKey), target: boxId(m.jobKey), type: 'bezier', style: afterStyle }))
+	);
+	// Svelte Flow needs a parent before its children.
+	return { nodes: [...boxNodes, ...cards], edges: [...lineage, ...after] };
+}
+
+/**
+ * Lays the graph out left to right. Cards use their measured size; boxes are sized to wrap their
+ * cards, and child positions are relative to their box, as Svelte Flow expects.
+ */
+export function layoutJobGraph(nodes: Node[], edges: Edge[]): Node[] {
+	const g = new dagre.graphlib.Graph({ compound: true });
+	g.setGraph({ rankdir: 'LR', ranksep: 64, nodesep: 24 + HEADER, marginx: 24, marginy: 24 });
+	g.setDefaultEdgeLabel(() => ({}));
+
+	const size = (n: Node) => ({ width: n.measured?.width ?? NODE_WIDTH, height: n.measured?.height ?? NODE_HEIGHT });
+	const children = new Map<string, Node[]>();
+	for (const n of nodes) if (n.parentId) children.set(n.parentId, [...(children.get(n.parentId) ?? []), n]);
+
+	for (const n of nodes) {
+		if (n.type === 'job') {
+			// An empty box is laid out as a plain node; dagre sizes boxes with children itself.
+			if (children.has(n.id)) g.setNode(n.id, {});
+			else g.setNode(n.id, { width: NODE_WIDTH + 2 * PAD, height: EMPTY_HEIGHT });
+		} else g.setNode(n.id, size(n));
+	}
+	for (const n of nodes) if (n.parentId) g.setParent(n.id, n.parentId);
+
+	// Dagre can't route edges to a box with children, so an `after` edge orders the boxes through their first cards.
+	const anchor = (id: string) => children.get(id)?.[0]?.id ?? id;
+	for (const e of edges) g.setEdge(anchor(e.source), anchor(e.target));
+
+	dagre.layout(g);
+
+	const topLeft = (n: Node) => {
+		const p = g.node(n.id);
+		const { width, height } = size(n);
+		return { x: p.x - width / 2, y: p.y - height / 2, width, height };
+	};
+	const boxes = new Map<string, { x: number; y: number; width: number; height: number }>();
+	for (const n of nodes) {
+		if (n.type !== 'job') continue;
+		const kids = (children.get(n.id) ?? []).map(topLeft);
+		if (!kids.length) {
+			const p = g.node(n.id);
+			boxes.set(n.id, { x: p.x - (NODE_WIDTH + 2 * PAD) / 2, y: p.y - EMPTY_HEIGHT / 2, width: NODE_WIDTH + 2 * PAD, height: EMPTY_HEIGHT });
+			continue;
+		}
+		const left = Math.min(...kids.map((k) => k.x)) - PAD;
+		const top = Math.min(...kids.map((k) => k.y)) - PAD - HEADER;
+		const right = Math.max(...kids.map((k) => k.x + k.width)) + PAD;
+		const bottom = Math.max(...kids.map((k) => k.y + k.height)) + PAD;
+		boxes.set(n.id, { x: left, y: top, width: right - left, height: bottom - top });
+	}
+
+	return nodes.map((n) => {
+		const box = boxes.get(n.id);
+		if (box) return { ...n, position: { x: box.x, y: box.y }, width: box.width, height: box.height };
+		const own = topLeft(n);
+		const parent = n.parentId ? boxes.get(n.parentId) : undefined;
+		return { ...n, position: parent ? { x: own.x - parent.x, y: own.y - parent.y } : { x: own.x, y: own.y } };
+	});
+}

ui/src/routes/admin/check-access/+page.svelte

@@ -27,10 +27,7 @@
 <AppTopbar title="Check access" />
 
 <div class="min-h-0 flex-1 overflow-auto px-6 pt-5 pb-8">
-	<p class="lede">
-		Can someone do something, and why? Pick a person, an action and where. Or paste the denial ID from a “permission denied” message to check that request again.
-		{#if !canCheckOthers}You can check your own access; checking others needs “Give roles” or “See the audit log”.{/if}
-	</p>
+	{#if !canCheckOthers}<p class="lede">You can check your own access; checking others needs “Give roles” or “See the audit log”.</p>{/if}
 	<div class="max-w-[1100px]">
 		<AccessChecker principals={canCheckOthers ? (accounts.data ?? []) : []} {initial} />
 	</div>

ui/src/routes/admin/people/+page.svelte

@@ -44,12 +44,6 @@
 {#if !allowed}
 	<NoAccess what="people" requires={['user.manage', 'role.assign']} />
 {:else}
-	<div class="intro">
-		<p>
-			Everyone who has signed in, with what they can do in each location. People sign in with single sign-on, or <code>sqb login</code> on the command line.
-			{#if opensPeople}Open someone to see their effective access and why.{/if}
-		</p>
-	</div>
 	<ListToolbar bind:query={people.query} placeholder="Search people and roles" total="{people.shown.length} of {people.accounts.length} people">
 		{#snippet filters()}
 			<ChoiceFilter label="Role" bind:value={people.role} options={people.roleNames.map((r) => ({ value: r, label: r }))} all="Any role" />
@@ -116,13 +110,6 @@
 {/if}
 
 <style>
-	.intro {
-		flex: 0 0 auto;
-		max-width: 900px;
-		padding: 14px 20px 10px;
-		font-size: 12.5px;
-		line-height: 1.55;
-	}
 	code {
 		font-family: var(--font-mono);
 		font-size: 12px;

ui/src/routes/admin/roles/roles-tab.svelte

@@ -11,10 +11,6 @@
 </script>
 
 <div class="px-6 pt-5 pb-8">
-	<p class="lede">
-		Roles in <b>{app.location?.name}</b>. Built-in roles act on every node; custom roles add actions on chosen nodes. Custom roles come from the project's roles file and are
-		read-only here.
-	</p>
 	<table class="sb-list framed">
 		<thead>
 			<tr>
@@ -46,16 +42,6 @@
 </div>
 
 <style>
-	.lede {
-		max-width: 860px;
-		margin-bottom: 18px;
-		color: var(--muted-foreground);
-		font-size: 12.5px;
-	}
-	.lede b {
-		color: var(--foreground);
-		font-weight: 500;
-	}
 	.framed {
 		border-top: 1px solid var(--border);
 	}