Skip to content

Commit d9abd83

Browse files
kvlongekvlonge
authored
feat: add microbatch coverage endpoint (#1319)
* feat: read check evidence through the control API People with view.evidence on a check's model can read the sample rows stored with one of its results. Columns are the rows' keys in sorted order, the order the warehouse stores them in, and a row without a key holds null there. Every read is audited, and a read whose audit cannot be written discloses nothing. The SQL and evidence reads now share one authorize, read and audit path. * feat: read maintenance history through the control API People with view on a target can read its recorded model moves, every step of their old-name views, and janitor cleanups, newest moves first. Filtering by model returns only moves into that model and their views, using new indexes on the destination model and the move's event. Check and maintenance reads now live in their own warehouse-history domain, apart from metadata ingestion. * feat: read microbatch coverage through the control API People with view on a target can read one model's recorded microbatch coverage: its completed segments, the holes between completed partitions, and segments a newer replay requirement still needs rebuilt at its version. Coverage is projected from the mirrored completion and replay-requirement facts of the model's newest relation and generation, with the same projection the executor uses; cursor bounds sort by value, so offset timestamps order by instant rather than text. Coverage is tracked only for concurrent microbatch models, so other models read as null. * test: keep the worker contract's scheduler lease alive The worker contract held one 30 second scheduler lease without renewing it, so on a slow runner its second claim came after expiry and was fenced. Renew it in the background, as a running scheduler does.
1 parent a99ff90 commit d9abd83

20 files changed

Lines changed: 1095 additions & 6 deletions

File tree

‎Cargo.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ order = [
4242
"sqlbuild-config",
4343
"sqlbuild-host",
4444
"sqlbuild-recall",
45-
"sqlbuild-orchestration",
4645
"sqlbuild-model-config",
46+
"sqlbuild-orchestration",
4747
"sqlbuild-discovery",
4848
"sqlbuild-render",
4949
"sqlbuild-scopes",

‎benchmarks/duckshop/edge_cases/orchestration/microbatch_coverage.json‎

Lines changed: 368 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,368 @@
1+
{
2+
"microbatches": [
3+
{
4+
"cursor_column": "ordered_at",
5+
"cursor_type": "timestamp",
6+
"batch_size": "1 hour",
7+
"run_start": "2026-01-01 00:00:00+00",
8+
"run_end": "2026-01-01T08:00:00+03:00:00",
9+
"event_id": "orders-old-generation",
10+
"created_at": "2026-01-01T00:30:00Z",
11+
"observed_at": "2026-01-01T00:30:00Z",
12+
"record_type": "partition_completion",
13+
"partition_start": "2026-01-01T10:00:00+00:00",
14+
"partition_end": "2026-01-01T11:00:00+00:00",
15+
"execution_run_id": "run-0",
16+
"model_version_hash": "orders-v1",
17+
"physical_generation_id": "orders-generation-0"
18+
},
19+
{
20+
"cursor_column": "ordered_at",
21+
"cursor_type": "timestamp",
22+
"batch_size": "1 hour",
23+
"run_start": "2026-01-01 00:00:00+00",
24+
"run_end": "2026-01-01T08:00:00+03:00:00",
25+
"event_id": "orders-other-scope",
26+
"created_at": "2026-01-01T00:35:00Z",
27+
"observed_at": "2026-01-01T00:35:00Z",
28+
"record_type": "partition_completion",
29+
"partition_start": "2026-01-01T12:00:00+00:00",
30+
"partition_end": "2026-01-01T13:00:00+00:00",
31+
"execution_run_id": "run-0",
32+
"model_version_hash": "orders-v1",
33+
"scope_key": "orders:main.orders_previous"
34+
},
35+
{
36+
"cursor_column": "ordered_at",
37+
"cursor_type": "timestamp",
38+
"batch_size": "1 hour",
39+
"run_start": "2026-01-01 00:00:00+00",
40+
"run_end": "2026-01-01T08:00:00+03:00:00",
41+
"event_id": "orders-older-requirement",
42+
"created_at": "2026-01-01T00:45:00Z",
43+
"observed_at": "2026-01-01T00:45:00Z",
44+
"record_type": "replay_requirement",
45+
"partition_start": null,
46+
"partition_end": null,
47+
"completion_type": null,
48+
"execution_run_id": "run-replay",
49+
"model_version_hash": "orders-v9",
50+
"replay_requirement_id": "orders-older-requirement",
51+
"required_model_version_hash": "orders-v9",
52+
"previous_model_version_hash": "orders-v8",
53+
"replay_policy": "full"
54+
},
55+
{
56+
"cursor_column": "ordered_at",
57+
"cursor_type": "timestamp",
58+
"batch_size": "1 hour",
59+
"run_start": "2026-01-01 00:00:00+00",
60+
"run_end": "2026-01-01T08:00:00+03:00:00",
61+
"event_id": "orders-batch-1",
62+
"created_at": "2026-01-01T01:30:00Z",
63+
"observed_at": "2026-01-01T01:30:00Z",
64+
"record_type": "partition_completion",
65+
"partition_start": "2026-01-01 00:00:00+00",
66+
"partition_end": "2026-01-01T09:00:00+08:00",
67+
"execution_run_id": "run-1",
68+
"model_version_hash": "orders-v1"
69+
},
70+
{
71+
"cursor_column": "ordered_at",
72+
"cursor_type": "timestamp",
73+
"batch_size": "1 hour",
74+
"run_start": "2026-01-01 00:00:00+00",
75+
"run_end": "2026-01-01T08:00:00+03:00:00",
76+
"event_id": "orders-batch-2-older",
77+
"created_at": "2026-01-01T01:00:00Z",
78+
"observed_at": "2026-01-01T01:00:00Z",
79+
"record_type": "partition_completion",
80+
"partition_start": "2026-01-01T01:00:00+00:00",
81+
"partition_end": "2026-01-01T02:00:00+00:00",
82+
"execution_run_id": "run-old",
83+
"model_version_hash": "orders-v1"
84+
},
85+
{
86+
"cursor_column": "ordered_at",
87+
"cursor_type": "timestamp",
88+
"batch_size": "1 hour",
89+
"run_start": "2026-01-01 00:00:00+00",
90+
"run_end": "2026-01-01T08:00:00+03:00:00",
91+
"event_id": "orders-batch-2",
92+
"created_at": "2026-01-01T02:30:00Z",
93+
"observed_at": "2026-01-01T02:30:00Z",
94+
"record_type": "partition_completion",
95+
"partition_start": "2026-01-01T09:00:00+08:00",
96+
"partition_end": "2026-01-01T02:00:00+00:00",
97+
"execution_run_id": "run-2",
98+
"model_version_hash": "orders-v1"
99+
},
100+
{
101+
"cursor_column": "ordered_at",
102+
"cursor_type": "timestamp",
103+
"batch_size": "1 hour",
104+
"run_start": "2026-01-01 00:00:00+00",
105+
"run_end": "2026-01-01T08:00:00+03:00:00",
106+
"event_id": "orders-batch-4",
107+
"created_at": "2026-01-01T04:30:00Z",
108+
"observed_at": "2026-01-01T04:30:00Z",
109+
"record_type": "partition_completion",
110+
"partition_start": "2026-01-01T03:00:00+00:00",
111+
"partition_end": "2026-01-01T04:00:00+00:00",
112+
"execution_run_id": "run-3",
113+
"model_version_hash": "orders-v2"
114+
},
115+
{
116+
"cursor_column": "ordered_at",
117+
"cursor_type": "timestamp",
118+
"batch_size": "1 hour",
119+
"run_start": "2026-01-01 00:00:00+00",
120+
"run_end": "2026-01-01T08:00:00+03:00:00",
121+
"event_id": "orders-batch-5",
122+
"created_at": "2026-01-01T05:30:00Z",
123+
"observed_at": "2026-01-01T05:30:00Z",
124+
"record_type": "synthetic_completion",
125+
"partition_start": "2026-01-01T04:00:00+00:00",
126+
"partition_end": "2026-01-01T05:00:00+00:00",
127+
"execution_run_id": "",
128+
"fingerprint_status": "unknown",
129+
"model_version_hash": null,
130+
"definition_hash": null,
131+
"synthetic_reason": "Retained rows",
132+
"unaccounted_policy": "synthesize",
133+
"observed_row_count": 12,
134+
"completion_type": "initial"
135+
},
136+
{
137+
"cursor_column": "ordered_at",
138+
"cursor_type": "timestamp",
139+
"batch_size": "1 hour",
140+
"run_start": "2026-01-01 00:00:00+00",
141+
"run_end": "2026-01-01T08:00:00+03:00:00",
142+
"event_id": "orders-batch-4-rerun",
143+
"created_at": "2026-01-01T05:45:00Z",
144+
"observed_at": "2026-01-01T05:45:00Z",
145+
"record_type": "partition_completion",
146+
"partition_start": "2026-01-01T03:00:00+00:00",
147+
"partition_end": "2026-01-01T04:00:00+00:00",
148+
"execution_run_id": "run-5",
149+
"model_version_hash": "orders-v2"
150+
},
151+
{
152+
"cursor_column": "ordered_at",
153+
"cursor_type": "timestamp",
154+
"batch_size": "1 hour",
155+
"run_start": "2026-01-01 00:00:00+00",
156+
"run_end": "2026-01-01T08:00:00+03:00:00",
157+
"event_id": "orders-batch-7",
158+
"created_at": "2026-01-01T05:50:00Z",
159+
"observed_at": "2026-01-01T05:50:00Z",
160+
"record_type": "synthetic_completion",
161+
"partition_start": "2026-01-01T06:00:00+00:00",
162+
"partition_end": "2026-01-01T07:00:00+00:00",
163+
"execution_run_id": "",
164+
"fingerprint_status": "unknown",
165+
"model_version_hash": null,
166+
"definition_hash": null,
167+
"synthetic_reason": "Retained rows",
168+
"unaccounted_policy": "synthesize",
169+
"observed_row_count": 12,
170+
"completion_type": "initial"
171+
},
172+
{
173+
"cursor_column": "ordered_at",
174+
"cursor_type": "timestamp",
175+
"batch_size": "1 hour",
176+
"run_start": "2026-01-01 00:00:00+00",
177+
"run_end": "2026-01-01T08:00:00+03:00:00",
178+
"event_id": "orders-requirement",
179+
"created_at": "2026-01-01T06:30:00Z",
180+
"observed_at": "2026-01-01T06:30:00Z",
181+
"record_type": "replay_requirement",
182+
"partition_start": null,
183+
"partition_end": null,
184+
"completion_type": null,
185+
"execution_run_id": "run-replay",
186+
"model_version_hash": "orders-v2",
187+
"replay_requirement_id": "orders-requirement",
188+
"required_model_version_hash": "orders-v2",
189+
"previous_model_version_hash": "orders-v1",
190+
"replay_policy": "full"
191+
},
192+
{
193+
"cursor_column": "ordered_at",
194+
"cursor_type": "timestamp",
195+
"batch_size": "1 hour",
196+
"run_start": "2026-01-01 00:00:00+00",
197+
"run_end": "2026-01-01T08:00:00+03:00:00",
198+
"event_id": "customers-batch-1",
199+
"created_at": "2026-01-01T01:30:00Z",
200+
"observed_at": "2026-01-01T01:30:00Z",
201+
"record_type": "partition_completion",
202+
"partition_start": "2026-01-01T00:00:00+00:00",
203+
"partition_end": "2026-01-01T01:00:00+00:00",
204+
"execution_run_id": "run-1",
205+
"model_version_hash": "customers-v1",
206+
"model_name": "customers",
207+
"scope_key": "customers:main.customers",
208+
"target_name": "customers"
209+
},
210+
{
211+
"cursor_column": "event_id",
212+
"cursor_type": "integer",
213+
"batch_size": "1",
214+
"run_start": "2",
215+
"run_end": "11",
216+
"event_id": "events-batch-2-older",
217+
"created_at": "2026-01-01T00:30:00Z",
218+
"observed_at": "2026-01-01T00:30:00Z",
219+
"record_type": "partition_completion",
220+
"partition_start": "2",
221+
"partition_end": "5",
222+
"execution_run_id": "run-old",
223+
"model_version_hash": "events-v1",
224+
"model_name": "events",
225+
"scope_key": "events:main.events",
226+
"target_name": "events",
227+
"physical_generation_id": "events-generation-1"
228+
},
229+
{
230+
"cursor_column": "event_id",
231+
"cursor_type": "integer",
232+
"batch_size": "1",
233+
"run_start": "2",
234+
"run_end": "11",
235+
"event_id": "events-batch-2",
236+
"created_at": "2026-01-01T01:00:00Z",
237+
"observed_at": "2026-01-01T01:00:00Z",
238+
"record_type": "partition_completion",
239+
"partition_start": "2",
240+
"partition_end": "9",
241+
"execution_run_id": "run-a",
242+
"model_version_hash": "events-v1",
243+
"model_name": "events",
244+
"scope_key": "events:main.events",
245+
"target_name": "events",
246+
"physical_generation_id": "events-generation-1"
247+
},
248+
{
249+
"cursor_column": "event_id",
250+
"cursor_type": "integer",
251+
"batch_size": "1",
252+
"run_start": "2",
253+
"run_end": "11",
254+
"event_id": "events-batch-9",
255+
"created_at": "2026-01-01T02:00:00Z",
256+
"observed_at": "2026-01-01T02:00:00Z",
257+
"record_type": "partition_completion",
258+
"partition_start": "9",
259+
"partition_end": "10",
260+
"execution_run_id": "run-b",
261+
"model_version_hash": "events-v1",
262+
"model_name": "events",
263+
"scope_key": "events:main.events",
264+
"target_name": "events",
265+
"physical_generation_id": "events-generation-1"
266+
},
267+
{
268+
"cursor_column": "event_id",
269+
"cursor_type": "integer",
270+
"batch_size": "1",
271+
"run_start": "2",
272+
"run_end": "11",
273+
"event_id": "events-batch-10",
274+
"created_at": "2026-01-01T03:00:00Z",
275+
"observed_at": "2026-01-01T03:00:00Z",
276+
"record_type": "partition_completion",
277+
"partition_start": "10",
278+
"partition_end": "11",
279+
"execution_run_id": "run-c",
280+
"model_version_hash": "events-v1",
281+
"model_name": "events",
282+
"scope_key": "events:main.events",
283+
"target_name": "events",
284+
"physical_generation_id": "events-generation-1"
285+
},
286+
{
287+
"cursor_column": "ordered_at",
288+
"cursor_type": "timestamp",
289+
"batch_size": "1 hour",
290+
"run_start": "2026-01-01T00:00:00+00:00",
291+
"run_end": "2026-01-01T02:00:00+00:00",
292+
"event_id": "payments-batch-0",
293+
"created_at": "2026-01-01T00:30:00Z",
294+
"observed_at": "2026-01-01T00:30:00Z",
295+
"record_type": "partition_completion",
296+
"partition_start": "2026-01-01T00:00:00+00:00",
297+
"partition_end": "2026-01-01T01:00:00+00:00",
298+
"execution_run_id": "run-p1",
299+
"model_version_hash": "payments-v1",
300+
"model_name": "payments",
301+
"scope_key": "payments:main.payments",
302+
"target_name": "payments",
303+
"physical_generation_id": "payments-generation-1"
304+
},
305+
{
306+
"cursor_column": "ordered_at",
307+
"cursor_type": "timestamp",
308+
"batch_size": "1 hour",
309+
"run_start": "2026-01-01T00:00:00+00:00",
310+
"run_end": "2026-01-01T02:00:00+00:00",
311+
"event_id": "payments-batch-1",
312+
"created_at": "2026-01-01T00:40:00Z",
313+
"observed_at": "2026-01-01T00:40:00Z",
314+
"record_type": "partition_completion",
315+
"partition_start": "2026-01-01T01:00:00+00:00",
316+
"partition_end": "2026-01-01T02:00:00+00:00",
317+
"execution_run_id": "run-p2",
318+
"model_version_hash": "payments-v1",
319+
"model_name": "payments",
320+
"scope_key": "payments:main.payments",
321+
"target_name": "payments",
322+
"physical_generation_id": "payments-generation-1"
323+
},
324+
{
325+
"cursor_column": "ordered_at",
326+
"cursor_type": "timestamp",
327+
"batch_size": "1 hour",
328+
"run_start": "2026-01-01T00:00:00+00:00",
329+
"run_end": "2026-01-01T02:00:00+00:00",
330+
"event_id": "payments-requirement",
331+
"created_at": "2026-01-01T01:00:00Z",
332+
"observed_at": "2026-01-01T01:00:00Z",
333+
"record_type": "replay_requirement",
334+
"partition_start": null,
335+
"partition_end": null,
336+
"completion_type": null,
337+
"execution_run_id": "run-replay",
338+
"model_version_hash": "payments-v2",
339+
"replay_requirement_id": "payments-requirement",
340+
"required_model_version_hash": "payments-v2",
341+
"previous_model_version_hash": "payments-v1",
342+
"replay_policy": "full",
343+
"model_name": "payments",
344+
"scope_key": "payments:main.payments",
345+
"target_name": "payments",
346+
"physical_generation_id": "payments-generation-1"
347+
},
348+
{
349+
"cursor_column": "ordered_at",
350+
"cursor_type": "timestamp",
351+
"batch_size": "1 hour",
352+
"run_start": "2026-01-01T00:00:00+00:00",
353+
"run_end": "2026-01-01T02:00:00+00:00",
354+
"event_id": "payments-batch-0-v3",
355+
"created_at": "2026-01-01T02:00:00Z",
356+
"observed_at": "2026-01-01T02:00:00Z",
357+
"record_type": "partition_completion",
358+
"partition_start": "2026-01-01T00:00:00+00:00",
359+
"partition_end": "2026-01-01T01:00:00+00:00",
360+
"execution_run_id": "run-p3",
361+
"model_version_hash": "payments-v3",
362+
"model_name": "payments",
363+
"scope_key": "payments:main.payments",
364+
"target_name": "payments",
365+
"physical_generation_id": "payments-generation-1"
366+
}
367+
]
368+
}

0 commit comments

Comments
 (0)