diff --git a/ui/src/lib/api/client.ts b/ui/src/lib/api/client.ts index f4349b7da0..87ebd3ac67 100644 --- a/ui/src/lib/api/client.ts +++ b/ui/src/lib/api/client.ts @@ -64,7 +64,8 @@ export interface ApiClient extends PlatformApi { overview(scope: Scope): Promise; /** Every node in the current snapshot with its build state; also the lineage graph. */ listCatalogNodes(scope: Scope): Promise; - getCatalogNode(scope: Scope, key: string): Promise; + /** `listing` is a current `listCatalogNodes` result the caller already holds, so it isn't read again. */ + getCatalogNode(scope: Scope, key: string, listing?: CatalogNode[]): Promise; /** The snapshot's column lineage around one node. */ getColumnLineage(scope: Scope, key: string): Promise; /** Every audit and check with its latest result. */ diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 5ae4b62c85..89f9f03210 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -12,7 +12,7 @@ import { overviewOf } from './http-overview'; import { toPlatformEvents, toPlatformHealth } from './http-problems'; import { roleDetailOf, toRoleDefinition } from './http-roles'; import { jobDetailOf, runWindowOf, toJob, toTick, toTrigger, triggerDetailOf } from './http-jobs'; -import { RUN_WINDOW, toAdhocCheck, runDetailOf, runRowPage, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs'; +import { RUN_WINDOW, groupRunNodes, toAdhocCheck, runDetailOf, runRowPage, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs'; import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals'; import type { AuthMode, Bootstrap, Capability, Location, PlatformEvent, Scope, WarehouseSync } from './types'; import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, LocationSummary, PlatformHealth, MaintenanceHistory, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform'; @@ -259,8 +259,8 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt if (!found) return null; const { run, wire, listed } = found; const owners = [run, ...listed.filter((r) => r.parentRunId === run.id)]; - const nodes = (await Promise.all(owners.map(async (owner) => (await runNodes(scope, owner.id)).map((n) => toRunNode(n, owner))))).flat(); - return runDetailOf(run, wire, listed, nodes); + const [own, ...children] = await Promise.all(owners.map(async (owner) => (await runNodes(scope, owner.id)).map((n) => toRunNode(n, owner)))); + return runDetailOf(run, wire, listed, groupRunNodes(own, children.flat())); }, async listRunEvents(scope: Scope, runId: string, query: { after?: number; limit?: number } = {}): Promise { const limit = query.limit ?? 1000; @@ -502,8 +502,8 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt if (!data) throw new ApiError(errorText(error, response), response.status); return toColumnLineage(data, nodes); }, - async getCatalogNode(scope: Scope, ref: string): Promise { - const nodes = await implemented.listCatalogNodes(scope); + async getCatalogNode(scope: Scope, ref: string, listing?: CatalogNode[]): Promise { + const nodes = listing ?? (await implemented.listCatalogNodes(scope)); const node = findNode(nodes, ref); if (!node) return null; const { locationId, target } = parseLocationKey(scope.locationId); diff --git a/ui/src/lib/api/http-runs.test.ts b/ui/src/lib/api/http-runs.test.ts index a3ffb3fa4b..97ec36118b 100644 --- a/ui/src/lib/api/http-runs.test.ts +++ b/ui/src/lib/api/http-runs.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { components } from '$control/schema.d.ts'; -import { runDetailOf, runRowPage, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunStatements, toRunOutput } from './http-runs'; +import { groupRunNodes, runDetailOf, runRowPage, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunStatements, toRunOutput } from './http-runs'; type RunRecord = components['schemas']['RunRecord']; type W = components['schemas']; @@ -176,3 +176,16 @@ describe('run Recall reads', () => { }); }); + +describe('group run nodes', () => { + it("keeps each job run's node and the group's own row only where no job run has the node", () => { + const group = toRuns([wire('g', { kind: 'group', job_key: 'nightly' })], LOCATION)[0]; + const staging = toRuns([wire('s', { parent_run_id: 'g', job_key: 'staging' })], LOCATION)[0]; + const node = (key: string, status: string, owner = group) => toRunNode({ node_key: key, status } as W['RunNodeRecord'], owner); + const nodes = groupRunNodes([node('model:raw', 'success'), node('model:report', 'pending')], [node('model:raw', 'failed', staging)]); + expect(nodes.map((n) => [n.nodeKey, n.runId, n.status])).toEqual([ + ['model:report', 'g', 'pending'], + ['model:raw', 's', 'failed'] + ]); + }); +}); diff --git a/ui/src/lib/api/http-runs.ts b/ui/src/lib/api/http-runs.ts index 3e91a7100b..a5e2c0375b 100644 --- a/ui/src/lib/api/http-runs.ts +++ b/ui/src/lib/api/http-runs.ts @@ -226,6 +226,15 @@ export function runDetailOf(run: Run, wire: Wire['RunRecord'], listed: Run[], no }; } +/** + * A run's nodes with its job runs' nodes. A group run's own rows repeat its job runs' nodes, so they + * stay only for nodes no job run has, such as those of members that never started. + */ +export function groupRunNodes(own: RunNode[], children: RunNode[]): RunNode[] { + const covered = new Set(children.map((n) => n.nodeKey)); + return [...own.filter((n) => !covered.has(n.nodeKey)), ...children]; +} + const KNOWN_EVENTS: ReadonlySet = new Set(RECALL_EVENT_TYPES); /** Scheduler facts are stored as `run_` events in the scheduler invocation. */ const SCHEDULER_EVENTS: ReadonlySet = new Set(SCHEDULER_FACTS.map((f) => `run_${f}`)); diff --git a/ui/src/lib/api/mock/build.ts b/ui/src/lib/api/mock/build.ts index 907039815c..099dd336bc 100644 --- a/ui/src/lib/api/mock/build.ts +++ b/ui/src/lib/api/mock/build.ts @@ -100,7 +100,7 @@ export class MockBuild { return this.#ctx.reply(SNAPSHOT_NODES.map((sn) => this.#catalogNode(states.get(sn.key)!, checks))); } - async getCatalogNode(scope: Scope, key: string): Promise { + async getCatalogNode(scope: Scope, key: string, _listing?: CatalogNode[]): Promise { const states = this.#states(scope); const state = states.get(key); if (!state) return this.#ctx.reply(null); diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index 2d73379a8d..4723e9ddc0 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -296,8 +296,8 @@ export class MockOperate { listRunEvents(scope: Scope, runId: string, query: { after?: number; limit?: number } = {}): Promise { const rec = this.#require(runId); const now = Date.now(); - const all = [rec, ...this.#children(rec)] - .flatMap((r) => execution(r, this.#nodesOf(r, now)).events) + // Each run's own, as the server returns; the run page gathers a group's job runs itself. + const all = execution(rec, this.#nodesOf(rec, now)).events .sort((a, b) => a.occurredAt.localeCompare(b.occurredAt) || a.storageId - b.storageId) .map((e, i) => ({ ...e, storageId: i + 1 })); const after = query.after ?? 0; @@ -310,16 +310,14 @@ export class MockOperate { listRunStatements(scope: Scope, runId: string): Promise { const rec = this.#require(runId); const now = Date.now(); - return this.#ctx.reply([rec, ...this.#children(rec)].flatMap((r) => execution(r, this.#nodesOf(r, now), now).statements)); + return this.#ctx.reply(execution(rec, this.#nodesOf(rec, now), now).statements); } getStatementSql(scope: Scope, runId: string, statementId: string): Promise { this.#need('view.sql'); const rec = this.#require(runId); const now = Date.now(); - const statement = [rec, ...this.#children(rec)] - .flatMap((r) => execution(r, this.#nodesOf(r, now), now).statements) - .find((st) => st.id === statementId); + const statement = execution(rec, this.#nodesOf(rec, now), now).statements.find((st) => st.id === statementId); if (!statement) throw new ApiError(`Unknown statement ${statementId}`, 404); return this.#ctx.reply({ statementId, sql: statement.sqlRecorded ? sqlFor(statement.nodeKey, statement.kind, this.#ctx.target(scope)) : null }); } @@ -327,7 +325,7 @@ export class MockOperate { listRunOutput(scope: Scope, runId: string): Promise { const rec = this.#require(runId); const now = Date.now(); - return this.#ctx.reply([rec, ...this.#children(rec)].flatMap((r) => execution(r, this.#nodesOf(r, now)).output)); + return this.#ctx.reply(execution(rec, this.#nodesOf(rec, now)).output); } listQueue(scope: Scope): Promise { diff --git a/ui/src/lib/components/app-topbar.svelte b/ui/src/lib/components/app-topbar.svelte index fafbf2725d..a11efe71df 100644 --- a/ui/src/lib/components/app-topbar.svelte +++ b/ui/src/lib/components/app-topbar.svelte @@ -78,5 +78,8 @@ .app-topbar { background: color-mix(in srgb, var(--background) 94%, transparent); backdrop-filter: blur(12px); + /* The blur makes the bar its own layer; this keeps it and its popovers, such as Run now, above the page. */ + position: relative; + z-index: 20; } diff --git a/ui/src/lib/lineage/job-graph.test.ts b/ui/src/lib/lineage/job-graph.test.ts new file mode 100644 index 0000000000..4745efd969 --- /dev/null +++ b/ui/src/lib/lineage/job-graph.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; +import type { Node } from '@xyflow/svelte'; +import type { CatalogNode } from '$lib/api'; +import { boxId, jobGraph, layoutJobGraph } from './job-graph'; + +const node = (key: string, upstream: string[] = []) => ({ key, kind: 'table', upstream, owner: [], tags: [], stale: null, checks: null }) as unknown as CatalogNode; +const measured = (nodes: Node[]) => nodes.map((n) => (n.type === 'job' ? n : { ...n, measured: { width: 236, height: 61 } })); + +// staging builds raw → stg; orders reads stg; reports waits for orders only through `after`. +const catalog = [node('model:raw'), node('model:stg', ['model:raw']), node('model:orders', ['model:stg']), node('model:report')]; +const all = catalog.map((n) => n.key); +const boxes = [ + { jobKey: 'staging', nodes: ['model:raw', 'model:stg'] }, + { jobKey: 'orders', nodes: ['model:orders', 'model:stg'] }, + { jobKey: 'reports', nodes: ['model:report'] } +]; +const members = [ + { jobKey: 'staging', upstream: [], stage: 0 }, + { jobKey: 'orders', upstream: [{ jobKey: 'staging', source: 'lineage' as const }], stage: 1 }, + { jobKey: 'reports', upstream: [{ jobKey: 'orders', source: 'after' as const }, { jobKey: 'staging', source: 'lineage' as const }], stage: 2 } +]; + +describe('job graph', () => { + it('puts each node in the first box that builds it and draws `after` only where no lineage links two boxes', () => { + const { nodes, edges } = jobGraph(all, catalog, boxes, members); + const parent = Object.fromEntries(nodes.filter((n) => n.type === 'asset').map((n) => [n.id, n.parentId])); + expect(parent).toEqual({ 'model:raw': boxId('staging'), 'model:stg': boxId('staging'), 'model:orders': boxId('orders'), 'model:report': boxId('reports') }); + // Boxes come before their cards, as Svelte Flow requires. + expect(nodes.findIndex((n) => n.id === boxId('reports'))).toBeLessThan(nodes.findIndex((n) => n.id === 'model:raw')); + expect(edges.map((e) => `${e.source}>${e.target}`).sort()).toEqual( + ['model:raw>model:stg', 'model:stg>model:orders', `${boxId('orders')}>${boxId('reports')}`, `${boxId('staging')}>${boxId('reports')}`].sort() + ); + }); + + it('sizes boxes around their cards, places cards inside them, and orders boxes by their dependencies', () => { + const { nodes, edges } = jobGraph(all, catalog, boxes, members); + const laid = layoutJobGraph(measured(nodes), edges); + const box = (k: string) => laid.find((n) => n.id === boxId(k))!; + for (const card of laid.filter((n) => n.parentId)) { + const b = laid.find((n) => n.id === card.parentId)!; + expect(card.position.x).toBeGreaterThan(0); + expect(card.position.y).toBeGreaterThan(0); + expect(card.position.x + 236).toBeLessThan(b.width!); + expect(card.position.y + 61).toBeLessThan(b.height!); + } + expect(box('staging').width).toBeGreaterThan(2 * 236); + expect(box('staging').position.x + box('staging').width!).toBeLessThan(box('orders').position.x); + expect(box('orders').position.x + box('orders').width!).toBeLessThan(box('reports').position.x); + }); + + it('draws a plain job without boxes', () => { + const { nodes } = jobGraph(['model:raw', 'model:stg'], catalog); + expect(nodes.map((n) => [n.id, n.type, n.parentId])).toEqual([ + ['model:raw', 'asset', undefined], + ['model:stg', 'asset', undefined] + ]); + }); +}); diff --git a/ui/src/lib/lineage/job-graph.ts b/ui/src/lib/lineage/job-graph.ts new file mode 100644 index 0000000000..8dec789f11 --- /dev/null +++ b/ui/src/lib/lineage/job-graph.ts @@ -0,0 +1,126 @@ +// A job's nodes as a lineage graph; for a group, each member job is a box around the nodes it builds. + +import dagre from '@dagrejs/dagre'; +import type { Edge, Node } from '@xyflow/svelte'; +import type { CatalogNode, JobGroupMember, RunRef } from '$lib/api'; +import { shortName } from '$lib/build-area/labels'; +import { cardData } from './catalog-graph'; + +const NODE_WIDTH = 236; +const NODE_HEIGHT = 61; +/** Room inside a member box: padding around its nodes plus its header. */ +const PAD = 14; +const HEADER = 30; +const EMPTY_HEIGHT = 44; + +export type JobBox = { jobKey: string; nodes: string[]; lastRun?: RunRef | null }; + +const edgeStyle = 'stroke: var(--sb-edge); stroke-width: 1.5;'; +const afterStyle = 'stroke: var(--sb-edge); stroke-width: 1.5; stroke-dasharray: 5 4;'; + +export function boxId(jobKey: string): string { + return `job:${jobKey}`; +} + +/** + * Cards for `nodes` and the lineage between them. With `boxes`, each node sits in the first box + * that builds it, and a declared `after` between two members with no lineage between them is a + * dashed edge between their boxes. + */ +export function jobGraph(nodes: string[], catalog: CatalogNode[], boxes: JobBox[] = [], members: JobGroupMember[] = []): { nodes: Node[]; edges: Edge[] } { + const byKey = new Map(catalog.map((n) => [n.key, n])); + const inJob = new Set(nodes); + const owner = new Map(); + for (const box of boxes) for (const n of box.nodes) if (inJob.has(n) && !owner.has(n)) owner.set(n, box.jobKey); + + const boxNodes: Node[] = boxes.map((b) => ({ + id: boxId(b.jobKey), + type: 'job', + position: { x: 0, y: 0 }, + width: NODE_WIDTH + 2 * PAD, + height: EMPTY_HEIGHT, + selectable: false, + data: { jobKey: b.jobKey, lastRun: b.lastRun ?? null, nodeCount: nodes.filter((n) => owner.get(n) === b.jobKey).length } + })); + const cards: Node[] = nodes.map((key) => { + const parent = owner.get(key); + const node = byKey.get(key); + return { + id: key, + type: 'asset', + position: { x: 0, y: 0 }, + ...(parent ? { parentId: boxId(parent), extent: 'parent' as const } : {}), + data: node ? cardData(node) : { name: key, label: shortName(key), kind: '', status: 'none' } + }; + }); + + const lineage: Edge[] = nodes.flatMap((key) => + (byKey.get(key)?.upstream ?? []).filter((up) => inJob.has(up)).map((up) => ({ id: `${up}-${key}`, source: up, target: key, type: 'bezier', style: edgeStyle })) + ); + const linked = new Set(lineage.map((e) => `${owner.get(e.source)}>${owner.get(e.target)}`)); + const after: Edge[] = members.flatMap((m) => + m.upstream + .filter((u) => !linked.has(`${u.jobKey}>${m.jobKey}`) && boxes.some((b) => b.jobKey === u.jobKey)) + .map((u) => ({ id: `after:${u.jobKey}-${m.jobKey}`, source: boxId(u.jobKey), target: boxId(m.jobKey), type: 'bezier', style: afterStyle })) + ); + // Svelte Flow needs a parent before its children. + return { nodes: [...boxNodes, ...cards], edges: [...lineage, ...after] }; +} + +/** + * Lays the graph out left to right. Cards use their measured size; boxes are sized to wrap their + * cards, and child positions are relative to their box, as Svelte Flow expects. + */ +export function layoutJobGraph(nodes: Node[], edges: Edge[]): Node[] { + const g = new dagre.graphlib.Graph({ compound: true }); + g.setGraph({ rankdir: 'LR', ranksep: 64, nodesep: 24 + HEADER, marginx: 24, marginy: 24 }); + g.setDefaultEdgeLabel(() => ({})); + + const size = (n: Node) => ({ width: n.measured?.width ?? NODE_WIDTH, height: n.measured?.height ?? NODE_HEIGHT }); + const children = new Map(); + for (const n of nodes) if (n.parentId) children.set(n.parentId, [...(children.get(n.parentId) ?? []), n]); + + for (const n of nodes) { + if (n.type === 'job') { + // An empty box is laid out as a plain node; dagre sizes boxes with children itself. + if (children.has(n.id)) g.setNode(n.id, {}); + else g.setNode(n.id, { width: NODE_WIDTH + 2 * PAD, height: EMPTY_HEIGHT }); + } else g.setNode(n.id, size(n)); + } + for (const n of nodes) if (n.parentId) g.setParent(n.id, n.parentId); + + // Dagre can't route edges to a box with children, so an `after` edge orders the boxes through their first cards. + const anchor = (id: string) => children.get(id)?.[0]?.id ?? id; + for (const e of edges) g.setEdge(anchor(e.source), anchor(e.target)); + + dagre.layout(g); + + const topLeft = (n: Node) => { + const p = g.node(n.id); + const { width, height } = size(n); + return { x: p.x - width / 2, y: p.y - height / 2, width, height }; + }; + const boxes = new Map(); + for (const n of nodes) { + if (n.type !== 'job') continue; + const kids = (children.get(n.id) ?? []).map(topLeft); + if (!kids.length) { + const p = g.node(n.id); + boxes.set(n.id, { x: p.x - (NODE_WIDTH + 2 * PAD) / 2, y: p.y - EMPTY_HEIGHT / 2, width: NODE_WIDTH + 2 * PAD, height: EMPTY_HEIGHT }); + continue; + } + const left = Math.min(...kids.map((k) => k.x)) - PAD; + const top = Math.min(...kids.map((k) => k.y)) - PAD - HEADER; + const right = Math.max(...kids.map((k) => k.x + k.width)) + PAD; + const bottom = Math.max(...kids.map((k) => k.y + k.height)) + PAD; + boxes.set(n.id, { x: left, y: top, width: right - left, height: bottom - top }); + } + + return nodes.map((n) => { + const box = boxes.get(n.id); + if (box) return { ...n, position: { x: box.x, y: box.y }, width: box.width, height: box.height }; + const own = topLeft(n); + const parent = n.parentId ? boxes.get(n.parentId) : undefined; + return { ...n, position: parent ? { x: own.x - parent.x, y: own.y - parent.y } : { x: own.x, y: own.y } }; + }); +} diff --git a/ui/src/routes/admin/check-access/+page.svelte b/ui/src/routes/admin/check-access/+page.svelte index 30e116f1e1..4c35ce5e5f 100644 --- a/ui/src/routes/admin/check-access/+page.svelte +++ b/ui/src/routes/admin/check-access/+page.svelte @@ -27,10 +27,7 @@
-

- Can someone do something, and why? Pick a person, an action and where. Or paste the denial ID from a “permission denied” message to check that request again. - {#if !canCheckOthers}You can check your own access; checking others needs “Give roles” or “See the audit log”.{/if} -

+ {#if !canCheckOthers}

You can check your own access; checking others needs “Give roles” or “See the audit log”.

{/if}
diff --git a/ui/src/routes/admin/people/+page.svelte b/ui/src/routes/admin/people/+page.svelte index 615f799646..1dd53c4b65 100644 --- a/ui/src/routes/admin/people/+page.svelte +++ b/ui/src/routes/admin/people/+page.svelte @@ -44,12 +44,6 @@ {#if !allowed} {:else} -
-

- Everyone who has signed in, with what they can do in each location. People sign in with single sign-on, or sqb login on the command line. - {#if opensPeople}Open someone to see their effective access and why.{/if} -

-
{#snippet filters()} ({ value: r, label: r }))} all="Any role" /> @@ -116,13 +110,6 @@ {/if} diff --git a/ui/src/routes/jobs/[name]/group-order.svelte b/ui/src/routes/jobs/[name]/group-order.svelte deleted file mode 100644 index a06b994e6a..0000000000 --- a/ui/src/routes/jobs/[name]/group-order.svelte +++ /dev/null @@ -1,101 +0,0 @@ - - -
- {#each stages as stage, i (i)} -
-
Step {i + 1}
- {#each stage as m (m.jobKey)} - - {m.jobKey} - {#if m.upstream.length} - {#each m.upstream as u (u.jobKey)} - - after {u.jobKey}{u.source ? ` · ${u.source === 'after' ? 'set in code' : 'uses its data'}` : ''} - - {/each} - {:else} - starts first - {/if} - - {/each} -
- {#if i < stages.length - 1}
→
{/if} - {/each} -
-{#if job.criticalNodes.length} -

- Critical nodes (if one fails, the whole group stops): - {#each job.criticalNodes as n, i (n)}{i ? ', ' : ''}{n}{/each} -

-{/if} - - diff --git a/ui/src/routes/jobs/[name]/job-box-node.svelte b/ui/src/routes/jobs/[name]/job-box-node.svelte new file mode 100644 index 0000000000..11bbb03a2d --- /dev/null +++ b/ui/src/routes/jobs/[name]/job-box-node.svelte @@ -0,0 +1,70 @@ + + +
+ +
+ {d.jobKey} + {d.nodeCount} node{d.nodeCount === 1 ? '' : 's'} + {#if d.lastRun} + + {runStatusLabel[d.lastRun.status]}{d.lastRun.startedAt ? ` · ${formatAgo(d.lastRun.startedAt, app.now)}` : ''} + + {/if} +
+ +
+ + diff --git a/ui/src/routes/jobs/[name]/job-graph.svelte b/ui/src/routes/jobs/[name]/job-graph.svelte new file mode 100644 index 0000000000..e7e70f04bb --- /dev/null +++ b/ui/src/routes/jobs/[name]/job-graph.svelte @@ -0,0 +1,74 @@ + + +
+ node.type === 'asset' && goto(`/catalog/${node.id}`)}> + + + + +
+ + diff --git a/ui/src/routes/jobs/[name]/job-view.svelte b/ui/src/routes/jobs/[name]/job-view.svelte index e61bec3718..f3cd427d4f 100644 --- a/ui/src/routes/jobs/[name]/job-view.svelte +++ b/ui/src/routes/jobs/[name]/job-view.svelte @@ -9,7 +9,7 @@ import { formatAgo, formatDuration, formatStamp } from '$lib/format/time'; import { formatNodeCounts, runOriginLabel, runStatusLabel, runStatusTone } from '$lib/status'; import { formatCost, onFailureHelp, onFailurePlain, overlapHelp, overlapPlain, shortRunId, slotLabel } from '$lib/operate/labels'; - import GroupOrder from './group-order.svelte'; + import JobGraph from './job-graph.svelte'; import LaunchForm from './launch-form.svelte'; let { name }: { name: string } = $props(); @@ -17,10 +17,23 @@ const job = createResource((client, scope) => client.getJob(scope, name), ['jobs', 'runs', 'snapshot']); const jobRuns = createResource((client, scope) => client.listRuns(scope, { jobKey: name, limit: 12 }), ['runs']); const triggers = createResource((client, scope) => client.listTriggers(scope), ['triggers', 'ticks']); - const nodes = createResource((client, scope) => client.listSnapshotNodes(scope), ['snapshot']); + const catalog = createResource((client, scope) => client.listCatalogNodes(scope), ['snapshot', 'runs', 'warehouse']); + // Each member's nodes and last run, so a group's graph can box every member job around what it builds. + // Reading `job.data` here reloads it with the job; it stays null until the job has loaded. + const members = createResource(async (client, scope) => { + const detail = job.data; + if (!detail) return null; + const keys = detail.groupMembers?.map((m) => m.jobKey) ?? []; + if (!keys.length) return { nodes: new Map(), lastRuns: new Map() }; + const jobs = await client.listJobs(scope); + const details = await Promise.all(keys.map((k) => client.getJob(scope, k))); + return { + nodes: new Map(keys.map((k, i) => [k, details[i]?.resolvedNodes ?? []])), + lastRuns: new Map(jobs.map((j) => [j.key, j.lastRun])) + }; + }, ['jobs', 'runs', 'snapshot']); const def = $derived(job.data ?? null); - const kindOf = $derived(new Map((nodes.data ?? []).map((n) => [n.key, n.kind]))); const jobTriggers = $derived((triggers.data ?? []).filter((t) => def?.triggerIds.includes(t.id))); const runs = $derived(jobRuns.data ?? []); const showCost = $derived(runs.some((r) => r.cost !== null)); @@ -39,15 +52,6 @@ } } - const KIND_TONE: Record = { - source: 'var(--sb-layer-source)', - loader: 'var(--sb-layer-source)', - seed: 'var(--sb-layer-staging)', - view: 'var(--sb-layer-staging)', - table: 'var(--sb-layer-marts)', - incremental: 'var(--sb-layer-intermediate)', - task: 'var(--sb-stale)' - }; const targetBlocked = $derived(!!def?.targets && !def.targets.includes(app.target?.name ?? '')); @@ -83,19 +87,20 @@ {#if def}
- {#if def.declaration} -
-

Code read-only; change it in the project and deploy

- -
- {/if} - - {#if def.kind === 'group' && def.groupMembers} -
-

Jobs in this group, in order from the data each job reads and writes, plus any after in the code

- -
- {/if} +
+

What it builds · {def.resolvedNodes.length} nodes {def.select ? `--select ${def.select}` : def.kind === 'group' ? 'each box is one of its jobs; arrows are the data each node reads' : ''}{def.exclude.length ? ` --exclude ${def.exclude.join(' ')}` : ''}

+ {#if catalog.data && (def.kind !== 'group' || members.data)} + + {:else} +

{catalog.error ?? members.error ?? 'Loading…'}

+ {/if} + {#if def.criticalNodes.length} +

+ Critical (if one fails, the whole group stops): + {#each def.criticalNodes as n, i (n)}{i ? ', ' : ''}{n}{/each} +

+ {/if} +

Recent runs

@@ -121,19 +126,12 @@ {/if}
-
-

What it builds · {def.resolvedNodes.length} nodes {def.select ? `--select ${def.select}` : def.kind === 'group' ? 'everything its jobs build' : ''}{def.exclude.length ? ` --exclude ${def.exclude.join(' ')}` : ''}

-
- {#each def.resolvedNodes as n (n)} - - - {n} - {kindOf.get(n) ?? ''} - {#if def.criticalNodes.includes(n)}critical{/if} - - {/each} -
-
+ {#if def.declaration} +
+

Code read-only; change it in the project and deploy

+ +
+ {/if}
- {d ? kindLine(d) : primary?.data.kind} - {#if d}{#if d.group}{d.group}{/if}{#if d.owner.length}{d.owner.join(', ')}{/if}{/if} + + {primary?.data.kind} + {#if primary?.data.group}{primary.data.group}{/if} + {#if (primary?.data.owner as string[] | undefined)?.length}{(primary?.data.owner as string[]).join(', ')}{/if}
{/if} -
+
{#if !scaleMode && !primary}

Select a node to see its details.

 {:else} @@ -651,7 +670,7 @@ {:else if !d} -

{detail.error ?? 'Loading…'}

 +

{detail.error ?? catalog.error ?? 'Loading…'}

 {:else if isSource}
@@ -1247,6 +1266,12 @@ .insp-body { display: flex; flex-direction: column; + transition: opacity 120ms; + } + /* A quick switch never dims; only one that takes longer than 150ms does. */ + .insp-body.switching { + opacity: 0.55; + transition: opacity 120ms 150ms; } .mini-run { display: grid; diff --git a/ui/src/routes/maintenance/+page.svelte b/ui/src/routes/maintenance/+page.svelte index 673f0f9d82..cf8a89ab6c 100644 --- a/ui/src/routes/maintenance/+page.svelte +++ b/ui/src/routes/maintenance/+page.svelte @@ -79,12 +79,6 @@ ] : []} > - {#snippet actions()} - - {#if tab === 'janitor'}Tables and views SQLBuild archived or deleted when cleaning up (sqb janitor).{:else if tab === 'views'}After a rename or move, a view under the old name keeps old queries working until its end date.{:else}Models renamed or moved to another schema (sqb rename, sqb mv).{/if} - Read-only; read back from the warehouse. - - {/snippet}
@@ -207,15 +201,6 @@
diff --git a/ui/src/routes/runs/[id]/events-tab.svelte b/ui/src/routes/runs/[id]/events-tab.svelte deleted file mode 100644 index 00786090f0..0000000000 --- a/ui/src/routes/runs/[id]/events-tab.svelte +++ /dev/null @@ -1,196 +0,0 @@ - - -
-
- {#each LEVELS as l (l.value)} - - {/each} -
-
- {#each EVENT_CATEGORIES as c (c.value)} - - {/each} -
- -
- - - - - {#if grouped}{/if} - - - - - - - {#if grouped}{/if} - - - - - - {#each shown as ev (ev.id)} - - - {#if grouped}{/if} - - - - {:else} - - {/each} - -
TimeJob runNodeWhat happened
{clockTime(ev.occurredAt)}{shortRunId(ev.runId)}{ev.nodeKey ?? ''} - - {eventSummary(ev)} -
{events.length ? 'No events match these filters.' : 'No events yet.'}
- - diff --git a/ui/src/routes/runs/[id]/failures-panel.svelte b/ui/src/routes/runs/[id]/failures-panel.svelte index bbaa7f3ea1..610ec45a23 100644 --- a/ui/src/routes/runs/[id]/failures-panel.svelte +++ b/ui/src/routes/runs/[id]/failures-panel.svelte @@ -1,4 +1,5 @@ + +
+
+ {#each LEVELS as l (l.value)} + + {/each} +
+
+ {#each present as c (c.value)} + + {/each} +
+ + +
+ + + + + {#if grouped}{/if} + + + + + + + {#if grouped}{/if} + + + + + + {#each shown as e (e.id)} + + + {#if grouped}{/if} + + + + {:else} + + {/each} + +
TimeJob runNodeWhat happened
{clockTime(e.at)}{shortRunId(e.runId)}{e.nodeKey ?? e.stream ?? 'run'} + {#if e.stream} + {@const lines = e.text.split('\n')} + {open.has(e.id) || lines.length <= FOLD ? e.text : lines.slice(0, FOLD).join('\n')} + {#if lines.length > FOLD}{/if} + {:else} + {e.text} + {/if} + {#if e.statementId && e.nodeKey && onstatement && e.category === 'queries'} + + {/if} +
{entries.length ? 'Nothing in the log matches these filters.' : 'Nothing logged yet.'}
+ + diff --git a/ui/src/routes/runs/[id]/node-panel.svelte b/ui/src/routes/runs/[id]/node-panel.svelte index b1443646f3..bd86d8c3e4 100644 --- a/ui/src/routes/runs/[id]/node-panel.svelte +++ b/ui/src/routes/runs/[id]/node-panel.svelte @@ -3,12 +3,20 @@ import { shortRunId } from '$lib/operate/labels'; import type { RunDetailState } from './state.svelte'; import type { NodeGroupKey } from './types'; - import { groupNodes, notRunReason } from './utils'; + import { formatSpan, groupNodes, notRunReason } from './utils'; let { view }: { view: RunDetailState } = $props(); const groups = $derived(groupNodes(view.nodes)); - let closed = $state>(new Set(['succeeded'])); + // Small runs show every node; large ones start with the successes folded away. + let closed = $state>(new Set()); + let folded = false; + $effect(() => { + if (folded || !view.nodes.length) return; + folded = true; + if (view.nodes.length > 40) closed = new Set(['succeeded']); + }); + const took = (n: RunNode) => (n.startedAt && n.endedAt ? formatSpan((Date.parse(n.endedAt) - Date.parse(n.startedAt)) / 1000) : null); function toggle(key: NodeGroupKey) { const next = new Set(closed); @@ -37,6 +45,7 @@ {#if n.skippedBecauseOf} {@const why = n.skippedBecauseOf} @@ -63,6 +72,12 @@
diff --git a/ui/src/routes/runs/[id]/run-gantt.svelte b/ui/src/routes/runs/[id]/run-gantt.svelte index 800b582dd4..5b51aa91b5 100644 --- a/ui/src/routes/runs/[id]/run-gantt.svelte +++ b/ui/src/routes/runs/[id]/run-gantt.svelte @@ -76,11 +76,14 @@ gutterW = Math.min(GUTTER_MAX, Math.max(GUTTER_MIN, e.clientX - ganttEl.getBoundingClientRect().left)); } + // Rows run in start order, so the bars read as a cascade; nodes that never started go last. + const ordered = $derived([...nodes].sort((a, b) => (a.startedAt ?? '\uffff').localeCompare(b.startedAt ?? '\uffff'))); + type Row = { type: 'group'; group: GanttGroup; count: number } | { type: 'node'; node: RunNode }; const rows = $derived.by((): Row[] => { - if (groups.length < 2) return nodes.map((node) => ({ type: 'node', node })); + if (groups.length < 2) return ordered.map((node) => ({ type: 'node', node })); return groups.flatMap((group): Row[] => { - const own = nodes.filter((n) => n.runId === group.runId); + const own = ordered.filter((n) => n.runId === group.runId); return [{ type: 'group', group, count: own.length }, ...own.map((node): Row => ({ type: 'node', node }))]; }); }); diff --git a/ui/src/routes/runs/[id]/run-header.svelte b/ui/src/routes/runs/[id]/run-header.svelte index f6c78c83ff..02b3e07193 100644 --- a/ui/src/routes/runs/[id]/run-header.svelte +++ b/ui/src/routes/runs/[id]/run-header.svelte @@ -31,12 +31,19 @@ {#if run.jobKey}{run.jobKey}{:else}ad hoc: {what}{/if} - {runKindLabel[run.kind].toLowerCase()} - {run.origin === 'rerun' ? 'rerun' : runOriginLabel[run.origin].toLowerCase()}{#if run.triggerName}: {run.triggerName}{/if} + + {runKindLabel[run.kind].toLowerCase()} + {#if run.triggerId} + {runOriginLabel[run.origin].toLowerCase()}{#if run.triggerName}: {run.triggerName}{/if} + {:else if run.rerunOfRunId} + rerun of {shortRunId(run.rerunOfRunId)} + {:else} + {runOriginLabel[run.origin].toLowerCase()} + {/if}
{#if run.startedAt} - Started {formatAgo(run.startedAt, app.now)} + Started {formatStamp(run.startedAt)} ({formatAgo(run.startedAt, app.now)}) · {view.active ? 'running for ' : 'took '}{formatDuration(run.startedAt, run.endedAt, app.now)} {:else} @@ -50,8 +57,17 @@ {view.progress.done}/{view.progress.total} nodes {/if} + {#if view.detail?.provenance} + {@const p = view.detail.provenance} + · + version {p.snapshotGeneration} + {/if}
+ {#if view.detail?.provenance?.command} + {@const command = view.detail.provenance.command} +
$ {command}
+ {/if} {:else if view.detailRes.error}

Couldn't load this run: {view.detailRes.error}

{:else if view.detailRes.data === null} @@ -81,6 +97,19 @@ {#if view.actionError}{/if}