diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 91a87c7d96..37ba35a5fa 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -64,6 +64,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/admin/server": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readServerSettings"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/auth/cli/approve": { parameters: { query?: never; @@ -546,6 +562,38 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/plans": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["requestPlan"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/control/locations/{location_id}/targets/{target}/plans/{request_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPlan"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/runs": { parameters: { query?: never; @@ -722,6 +770,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/triggers/{trigger_id}/sensor": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonSensorState"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/triggers/{trigger_id}/ticks": { parameters: { query?: never; @@ -738,6 +802,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/platform/events": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["listPersonPlatformEvents"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/platform/problems": { parameters: { query?: never; @@ -1467,6 +1547,12 @@ export interface components { /** @enum {string} */ status: "timed_out"; }; + /** @description Where a declaration was written, relative to the project root. */ + DeclaredAt: { + /** Format: int32 */ + line: number; + path: string; + }; /** @description Local definition identity only, not upstream data or graph currency. */ DefinitionEvidence: { local_hash: string; @@ -1527,18 +1613,33 @@ export interface components { failed: components["schemas"]["Table"][]; failure?: string | null; }; + /** @description The control store's installation and schema, as recorded when it was created. */ + InstallationRecord: { + /** Format: date-time */ + installed_at: string; + /** Format: date-time */ + schema_applied_at: string; + /** @description Digest of the schema; a server starts only on the exact schema it creates. */ + schema_checksum: string; + storage: components["schemas"]["StorageKind"]; + }; /** @description One job or group as declared, for a person who may list the target; free-form setting values are withheld. */ JobDetailRecord: { concurrency_groups: string[]; /** @description Selected nodes whose failure stops a group. */ critical_nodes: string[]; + /** @description The cursor type every selected cursor node shares; absent when none has a cursor or types differ. */ + cursor_type?: string | null; + declared_at?: null | components["schemas"]["DeclaredAt"]; description: string; + exclude?: string | null; job_key: string; kind: components["schemas"]["RunKind"]; /** @description The groups this job belongs to. */ member_of: string[]; /** @description A group's member jobs in run order; empty for a job. */ members: components["schemas"]["JobMember"][]; + select?: string | null; selection: string[]; /** @description Command settings; `vars` and a `defer_*` value that is not a target name show as `[REDACTED]`, and text values pass through secret redaction. */ settings: { @@ -1546,6 +1647,8 @@ export interface components { }; /** Format: uuid */ snapshot_id: string; + /** @description Empty when the declaration applies to every target. */ + targets: string[]; }; JobMember: { job_key: string; @@ -1555,16 +1658,21 @@ export interface components { /** @description A job or group of the current published snapshot. Settings are left out of person reads. */ JobRecord: { concurrency_groups: string[]; + declared_at?: null | components["schemas"]["DeclaredAt"]; description: string; job_key: string; /** @description `job` or `group`; a group's selection is the union of its member jobs' selections. */ kind: components["schemas"]["RunKind"]; + /** @description A job's selectors as written; absent for a group and for snapshots published before they were recorded. */ + select?: string | null; selection: string[]; /** Format: uuid */ snapshot_id: string; }; /** @description A trusted operator names a published job; execution inputs are never supplied by the caller. */ JobRunRequest: { + /** @description Leave nodes out of the job with `sqb build` selectors. */ + exclude?: string[]; /** @description Rebuild every selected model from scratch; it rewrites history, as a window does. */ full_refresh?: boolean; job_key: string; @@ -1575,6 +1683,8 @@ export interface components { reason: string; /** Format: uuid */ run_id: string; + /** @description Narrow the job with `sqb build` selectors; only nodes already in the job can remain. */ + select?: string[]; target: string; window?: null | components["schemas"]["RunWindow"]; }; @@ -1646,6 +1756,12 @@ export interface components { evaluation: Record; /** @enum {string} */ kind: "evaluate_sensor"; + } | { + command: string[]; + /** @enum {string} */ + kind: "plan"; + selection: string[]; + settings: Record; }; LocationWorkClaim: { /** Format: uuid */ @@ -1666,6 +1782,10 @@ export interface components { /** @enum {string} */ kind: "sensor_evaluation"; outcome: components["schemas"]["CustomSensorOutcome"]; + } | { + document: Record; + /** @enum {string} */ + kind: "plan"; } | { /** @enum {string} */ kind: "superseded"; @@ -1710,6 +1830,26 @@ export interface components { * @enum {string} */ LoginMode: "password" | "trusted_proxy" | "oidc"; + /** @description A running calendar schedule whose last date is at most a day away, or which has none left. */ + LowCalendar: { + /** + * Format: date-time + * @description The calendar's last date; absent when none are left. + */ + horizon_end?: string | null; + /** Format: uuid */ + location_id: string; + location_name: string; + name: string; + /** + * Format: date-time + * @description When the location next works out the calendar's dates. + */ + refresh_at: string; + target: string; + /** Format: uuid */ + trigger_id: string; + }; /** @description Recorded moves, their old-name view steps and janitor cleanups, newest moves first. */ MaintenanceHistory: { /** @description Empty when the history is for one model; cleanups are not tied to models. */ @@ -1904,6 +2044,28 @@ export interface components { /** Format: uuid */ principal_id: string; }; + /** @description A stored platform event a person may see; the location target is absent for server-wide events. */ + PersonPlatformEvent: { + event: components["schemas"]["PlatformEvent"]; + /** Format: uuid */ + event_id: string; + /** Format: uuid */ + location_id?: string | null; + /** Format: date-time */ + occurred_at: string; + /** Format: int64 */ + storage_id: number; + target?: string | null; + }; + /** @description One scan of stored platform events, keeping those the person may see. */ + PersonPlatformEventPage: { + /** + * Format: int64 + * @description The `before` that continues the scan, absent once no older events remain; a page can be empty and still continue. + */ + before?: number | null; + events: components["schemas"]["PersonPlatformEvent"][]; + }; /** @description A role assigned to one person, everywhere or on one location and target. */ PersonRole: { /** Format: uuid */ @@ -1916,6 +2078,49 @@ export interface components { role_key: string; target?: string | null; }; + /** @description The control event position built-in conditions read from, and a custom sensor's own value after secret redaction; the value needs view.evidence. */ + PersonSensorCursor: { + custom?: string | null; + /** Format: int64 */ + event_storage_id: number; + }; + PersonSensorEvaluation: { + cursor_after?: null | components["schemas"]["PersonSensorCursor"]; + cursor_before: components["schemas"]["PersonSensorCursor"]; + definition_fingerprint: string; + /** Format: uuid */ + evaluation_id: string; + /** @description The failure code of a failed check. */ + failure_code?: string | null; + /** Format: date-time */ + finished_at?: string | null; + /** @description The latest phase: `requested` alone means the check is in flight; otherwise `completed`, `failed`, `timed_out` or `abandoned`. */ + phase: string; + /** Format: date-time */ + requested_at: string; + run_count: number; + }; + /** @description A run key is redacted and withheld like the custom cursor it usually derives from: empty without view.evidence. */ + PersonSensorRunRequest: { + /** Format: uuid */ + evaluation_id: string; + /** Format: date-time */ + queued_at: string; + /** Format: uuid */ + run_id: string; + run_key: string; + status: components["schemas"]["RunStatus"]; + }; + /** @description A sensor's committed position and recent checks, for a person who may list the target. */ + PersonSensorState: { + cursor?: null | components["schemas"]["PersonSensorCursor"]; + /** @description Fingerprint of the published definition; a check with another one ran older code. */ + definition_fingerprint: string; + /** @description Newest first. */ + evaluations: components["schemas"]["PersonSensorEvaluation"][]; + /** @description Runs the sensor requested, by run key, newest first. */ + run_requests: components["schemas"]["PersonSensorRunRequest"][]; + }; /** @description A person as administrators see them: how they sign in, their SSO groups and live sessions. */ PersonSummary: { /** @@ -1961,6 +2166,8 @@ export interface components { /** @description A trigger as people list it, with its newest tick so a list needs no read per trigger. */ PersonTriggerRecord: components["schemas"]["TriggerRecord"] & { latest_tick?: null | components["schemas"]["TickRecord"]; + /** @description A running schedule's next due times from `next_due_at` on, at most five; empty otherwise. */ + upcoming: string[]; }; /** @description Observed physical evidence is not a live warehouse-existence guarantee. */ PhysicalEvidence: { @@ -1972,11 +2179,60 @@ export interface components { /** @enum {string} */ status: "unavailable"; }; + /** @description A person's build command to plan on the location; the server freezes it like an ad-hoc run. */ + PlanRequest: { + command: string[]; + /** Format: uuid */ + request_id: string; + }; + PlatformEvent: { + /** @enum {string} */ + kind: "heartbeat"; + /** Format: uuid */ + replica_id: string; + } | { + /** @enum {string} */ + kind: "scheduler_no_leader"; + } | { + /** @enum {string} */ + kind: "tick_overdue"; + /** Format: date-time */ + scheduled_for: string; + /** Format: uuid */ + trigger_id: string; + } | { + /** Format: date-time */ + horizon_end?: string | null; + /** @enum {string} */ + kind: "calendar_horizon_low"; + /** Format: date-time */ + refresh_at: string; + /** Format: uuid */ + trigger_id: string; + } | { + /** @enum {string} */ + kind: "location_broken"; + /** Format: date-time */ + last_seen_at: string; + /** Format: uuid */ + location_id: string; + target: string; + } | { + /** Format: uuid */ + claim_id: string; + code: string; + /** @enum {string} */ + kind: "launch_failed"; + /** Format: uuid */ + run_id: string; + }; /** @enum {string} */ PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed"; /** @description Problems derived from live state when read; nothing here is stored. */ PlatformProblems: { broken_locations: components["schemas"]["BrokenLocation"][]; + /** @description Running calendar schedules whose dates run out within a day. */ + low_calendars?: components["schemas"]["LowCalendar"][]; overdue_ticks: components["schemas"]["OverdueTick"][]; scheduler_leader: boolean; /** @@ -2007,6 +2263,45 @@ export interface components { snapshot_id: string; sqb_version: string; }; + /** @description Why the scheduler cannot claim a queued run yet. Runs outside the listed location and target are counted, never named. */ + QueueWait: { + /** @enum {string} */ + kind: "group_parent"; + /** Format: uuid */ + run_id: string; + } | { + job_key: string; + /** @enum {string} */ + kind: "upstream_job"; + /** Format: uuid */ + run_id: string; + } | { + /** @enum {string} */ + kind: "job_running"; + /** Format: uuid */ + run_id: string; + } | { + /** @enum {string} */ + kind: "write_claim"; + node_key: string; + /** Format: int32 */ + other_nodes: number; + /** Format: uuid */ + run_id?: string | null; + } | { + /** Format: int32 */ + elsewhere: number; + holders: string[]; + /** @enum {string} */ + kind: "concurrency_limit"; + limit: string; + } | { + /** @enum {string} */ + kind: "earlier_run"; + limit: string; + /** Format: uuid */ + run_id?: string | null; + }; /** @description One invocation's records; the server owns its claim namespace, start time and version. */ RecallBatch: { invocation_id: string; @@ -2044,6 +2339,8 @@ export interface components { /** @description A retry derives its execution scope from a terminal run, never from caller selectors. */ RerunRequest: { from_failure: boolean; + /** @description Rerun only these nodes of the source run; each must be in its selection. */ + nodes?: string[]; /** Format: int64 */ priority: number; reason: string; @@ -2213,6 +2510,11 @@ export interface components { }; /** @enum {string} */ RunKind: "job" | "group" | "adhoc"; + /** @description The `sqb build` selectors one run of a job was narrowed with. */ + RunNarrowing: { + exclude: string[]; + select: string[]; + }; /** @description A node's outcome error in one run: redacted, bounded text and the project file of the node, never its SQL. */ RunNodeFailure: { error_code?: string | null; @@ -2256,6 +2558,16 @@ export interface components { stream: string; text: string; }; + /** @description A queued run's place in claim order within its location and target, and what holds it back. */ + RunQueueState: { + /** + * Format: int32 + * @description 1-based, in the order the scheduler claims: priority, then queue time. + */ + position: number; + /** @description Empty when nothing holds the run back and it starts on a free worker. */ + waits: components["schemas"]["QueueWait"][]; + }; RunRecord: { /** Format: date-time */ cancel_requested_at?: string | null; @@ -2269,6 +2581,7 @@ export interface components { launched_at?: string | null; /** Format: uuid */ location_id: string; + narrowing?: null | components["schemas"]["RunNarrowing"]; node_counts_json: { [key: string]: number; }; @@ -2277,6 +2590,7 @@ export interface components { parent_run_id?: string | null; /** Format: int64 */ priority: number; + queue?: null | components["schemas"]["RunQueueState"]; /** Format: date-time */ queued_at: string; /** @description The stored actor: `principal:` for a person, otherwise a system actor such as `system:scheduler`. */ @@ -2303,6 +2617,7 @@ export interface components { /** Format: uuid */ trigger_id?: string | null; trigger_name?: string | null; + window?: null | components["schemas"]["RunWindow"]; }; /** @description The authorized selection and pinned execution inputs are immutable once requested. */ RunRequest: { @@ -2313,6 +2628,7 @@ export interface components { kind: components["schemas"]["RunKind"]; /** Format: uuid */ location_id: string; + narrowing?: null | components["schemas"]["RunNarrowing"]; origin: components["schemas"]["RunOrigin"]; /** Format: uuid */ parent_run_id?: string | null; @@ -2451,13 +2767,45 @@ export interface components { kind: "custom"; }; SensorDefinition: { + declared_at?: null | components["schemas"]["DeclaredAt"]; default_status?: components["schemas"]["TriggerStatus"]; description: string; job_key: string; /** Format: int32 */ minimum_interval_seconds: number; + /** @description The targets the declaration is limited to; empty for every target. */ + targets?: string[]; when: components["schemas"]["SensorCondition"]; }; + /** @description How long the server keeps each kind of record. */ + ServerRetention: { + /** Format: int64 */ + audit_days: number; + /** Format: int64 */ + heartbeat_days: number; + /** Format: int64 */ + location_health_days: number; + /** Format: int64 */ + platform_alert_days: number; + /** Format: int64 */ + sensor_evaluation_days: number; + /** + * Format: int64 + * @description Publications kept per location target. + */ + snapshots_kept: number; + /** Format: int64 */ + statistics_days: number; + }; + /** @description The server's version, storage, sign-in and retention, for administrators. */ + ServerSettings: { + installation: components["schemas"]["InstallationRecord"]; + login_mode: components["schemas"]["LoginMode"]; + retention: components["schemas"]["ServerRetention"]; + /** @description Whether this server runs the scheduler; another server may hold the lease. */ + scheduler: boolean; + version: string; + }; SessionIdentity: { /** Format: date-time */ expires_at: string; @@ -2551,6 +2899,11 @@ export interface components { /** Format: uuid */ next_after?: string | null; }; + /** + * @description Where the control store keeps its data. + * @enum {string} + */ + StorageKind: "sqlite" | "postgresql"; /** * @description The tables Recall writes. * @enum {string} @@ -2613,12 +2966,15 @@ export interface components { /** @description Legacy schedule documents retain their stored shape; sensors add a distinct condition shape. */ TriggerDeclaration: components["schemas"]["TriggerDefinition"] | components["schemas"]["SensorDefinition"]; TriggerDefinition: { + declared_at?: null | components["schemas"]["DeclaredAt"]; default_status?: components["schemas"]["TriggerStatus"]; description: string; job_key: string; schedule: components["schemas"]["Schedule"]; /** Format: date-time */ starts_at?: string | null; + /** @description The targets the declaration is limited to; empty for every target. */ + targets?: string[]; }; TriggerRecord: { definition_json: components["schemas"]["TriggerDeclaration"]; @@ -2934,6 +3290,40 @@ export interface operations { }; }; }; + readServerSettings: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ServerSettings"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; approveCliLogin: { parameters: { query?: never; @@ -4090,10 +4480,105 @@ export interface operations { }; }; }; + requestPlan: { + parameters: { + query?: never; + header: { + "x-sqb-csrf": string; + }; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PlanRequest"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MutationResponse"]; + }; + }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; + readPlan: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + request_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": null | components["schemas"]["LocationWorkRecord"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; listPersonRuns: { parameters: { query?: { + /** @description Page size; defaults to 100, capped at 1000 */ limit?: number; + /** @description With before_run_id: continue after this run of the previous page */ + before_queued_at?: string; + before_run_id?: string; + /** @description List queued runs in claim order, each with its position and what it waits on */ + queued?: boolean; }; header?: never; path: { @@ -4104,6 +4589,7 @@ export interface operations { }; requestBody?: never; responses: { + /** @description Runs newest first, or the queue in claim order */ 200: { headers: { [name: string]: unknown; @@ -4112,6 +4598,13 @@ export interface operations { "application/json": components["schemas"]["RunRecord"][]; }; }; + /** @description Only one of the before_* pair was given, or a cursor with queued */ + 400: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; /** @description Session rejected */ 401: { headers: { @@ -4596,6 +5089,55 @@ export interface operations { }; }; }; + readPersonSensorState: { + parameters: { + query?: { + /** @description Maximum recent checks and run requests; defaults to 100, capped at 1000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + trigger_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The cursor, recent checks newest first and requested runs */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PersonSensorState"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Sensor not found for a person who may view the whole target */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; listPersonTicks: { parameters: { query?: { @@ -4644,6 +5186,38 @@ export interface operations { }; }; }; + listPersonPlatformEvents: { + parameters: { + query?: { + /** @description Continue after this storage ID of the previous page */ + before?: number; + /** @description Events to scan; defaults to 200, clamped to 1..500 */ + limit?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description One scan of stored platform events newest first, without heartbeats: server-wide ones, and those of targets the person lists */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PersonPlatformEventPage"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; readPersonPlatformProblems: { parameters: { query?: never; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/selection.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/selection.rs index dd307490d5..bfbbb8d73d 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/selection.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/selection.rs @@ -9,8 +9,8 @@ use sqlbuild_core::text::main::python_isspace::python_isspace; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use crate::dag_artifact::constants::{ - SELECTOR_KIND_CHECK, SELECTOR_KIND_NAME, SELECTOR_KIND_PATH, SELECTOR_KIND_SEPARATOR, - SELECTOR_KIND_TAG, SELECTOR_KIND_TEST, SELECTOR_MODELS_PREFIX, + SELECTOR_KIND_CHECK, SELECTOR_KIND_JOB, SELECTOR_KIND_NAME, SELECTOR_KIND_PATH, + SELECTOR_KIND_SEPARATOR, SELECTOR_KIND_TAG, SELECTOR_KIND_TEST, SELECTOR_MODELS_PREFIX, }; use crate::dag_artifact::errors::DagSelectionError; use crate::dag_artifact::models::{DagSelectionIndex, DagSelectionNode}; @@ -146,6 +146,12 @@ fn match_kind( "{SELECTOR_KIND_TEST}{SELECTOR_KIND_SEPARATOR}{value}" ))); } + SELECTOR_KIND_JOB => { + return Err(DagSelectionError::Input(format!( + "selector '{SELECTOR_KIND_JOB}{SELECTOR_KIND_SEPARATOR}{value}' selects a job; \ + select the job's assets by name, tag or path instead" + ))); + } SELECTOR_KIND_TAG => ids(&|node| node.tags.iter().any(|tag| tag == value)), SELECTOR_KIND_PATH => { let folder: String = value.replace('\\', "/"); diff --git a/crates/sqlbuild-analysis/src/dag_artifact/constants.rs b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs index 5de360110e..ea0c4a97f4 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/constants.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs @@ -32,6 +32,7 @@ pub(crate) const SELECTOR_KIND_TAG: &str = "tag"; pub(crate) const SELECTOR_KIND_PATH: &str = "path"; pub(crate) const SELECTOR_KIND_CHECK: &str = "check"; pub(crate) const SELECTOR_KIND_TEST: &str = "test"; +pub(crate) const SELECTOR_KIND_JOB: &str = "job"; pub(crate) const SELECTOR_KIND_SEPARATOR: &str = ":"; pub(crate) const SELECTOR_MODELS_PREFIX: &str = "models/"; /// The orchestrator kind every SQLBuild asset carries. diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs b/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs new file mode 100644 index 0000000000..329bbbc646 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs @@ -0,0 +1,51 @@ +//! Jobs and job groups resolved once on the graph, so `job:` selects what the job publishes. + +use std::collections::BTreeMap; + +use crate::graph::_helpers::selectors::{Keys, resolve}; +use crate::graph::constants::UNKNOWN_NAME_CODE; +use crate::graph::errors::SelectorError; +use crate::graph::models::{JobSelection, ProjectGraph}; + +/// Resolve every job on the graph without jobs attached, as publication does, then attach them. +pub(crate) fn attach( + mut graph: ProjectGraph, + catalog: BTreeMap, +) -> ProjectGraph { + graph.jobs = None; + let mut resolved: BTreeMap> = BTreeMap::new(); + for (name, selection) in &catalog { + let keys: Result = match selection { + JobSelection::Job { select, exclude } => { + resolve(&graph, select, exclude).map(|keys| keys.into_iter().collect()) + } + JobSelection::Resolved { keys } => Ok(keys.iter().cloned().collect()), + JobSelection::Group { .. } => continue, + }; + resolved.insert(name.clone(), keys); + } + for (name, selection) in &catalog { + if let JobSelection::Group { members } = selection { + let mut keys: Keys = Keys::new(); + let mut failure: Option = None; + for member in members { + match resolved.get(member) { + Some(Ok(member_keys)) => keys.extend(member_keys.iter().cloned()), + Some(Err(error)) => failure = failure.or_else(|| Some(error.clone())), + None => { + failure = failure.or_else(|| { + Some(SelectorError { + code: UNKNOWN_NAME_CODE, + message: format!("job group '{name}' names unknown job '{member}'"), + help: None, + }) + }); + } + } + } + resolved.insert(name.clone(), failure.map_or(Ok(keys), Err)); + } + } + graph.jobs = Some(resolved); + graph +} diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/mod.rs b/crates/sqlbuild-analysis/src/graph/_helpers/mod.rs index 08c264a145..32fb6b3e21 100644 --- a/crates/sqlbuild-analysis/src/graph/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/graph/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod closure; pub(crate) mod indexes; +pub(crate) mod jobs; pub(crate) mod layers; pub(crate) mod selectors; pub(crate) mod unique_ids; diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs b/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs index 07740070f8..9ff9af6d87 100644 --- a/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs +++ b/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs @@ -5,7 +5,7 @@ use std::collections::BTreeSet; use crate::assembly::project::types::ObjectKey; use crate::graph::constants::{ EMPTY_SELECTOR_CODE, EMPTY_VALUE_CODE, EXPANSION_MARKER, FOLDER_SEPARATOR, - INTERSECTION_SEPARATOR, KIND_SEPARATOR, MISPLACED_MARKER_CODE, MISSING_NAME_CODE, + INTERSECTION_SEPARATOR, JOB_KIND, KIND_SEPARATOR, MISPLACED_MARKER_CODE, MISSING_NAME_CODE, MODEL_RESOURCE, MODEL_ROOT, MODEL_ROOT_PREFIX, NAME_KIND, PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PATH_SELECTOR_CODE, PATH_SEPARATOR, PATTERN_CHARACTERS, PLANNER_DEFAULT_CODE, SEED_RESOURCE, SOURCE_RESOURCE, SQL_FILE_SUFFIX, SUGGESTION_CUTOFF, SUGGESTION_LIMIT, @@ -161,10 +161,41 @@ fn path(graph: &ProjectGraph, names: (&str, &str), expand: (bool, bool)) -> Reso } fn named(graph: &ProjectGraph, name: &str) -> Result { - graph - .key_named(name) - .cloned() - .ok_or_else(|| error(UNKNOWN_NAME_CODE, format!("unknown selector name '{name}'"))) + if let Some(key) = graph.key_named(name) { + return Ok(key.clone()); + } + let job: bool = name.starts_with(&format!("{JOB_KIND}{KIND_SEPARATOR}")) + || graph + .jobs + .as_ref() + .is_some_and(|jobs| jobs.contains_key(name)); + Err(if job { + error( + PATH_SELECTOR_CODE, + format!("path selector endpoint '{name}' is a job; path selectors join two nodes"), + ) + } else { + error(UNKNOWN_NAME_CODE, format!("unknown selector name '{name}'")) + }) +} + +/// Whether a `job:` part or an unknown bare name in `selectors` needs jobs; paths never do. +pub(crate) fn needs_jobs(graph: &ProjectGraph, selectors: &[String], also_known: &[&str]) -> bool { + selectors + .iter() + .flat_map(|selector| selector.split(char::is_whitespace)) + .flat_map(|token| token.split(INTERSECTION_SEPARATOR)) + .filter(|part| !part.is_empty()) + .any(|part| match parse(part) { + Ok(ParsedSelector::Kind { kind, value, .. }) => { + kind == JOB_KIND + || (kind == NAME_KIND + && !is_pattern(&value) + && graph.key_named(&value).is_none() + && !also_known.contains(&value.as_str())) + } + _ => false, + }) } pub(crate) fn parse(raw: &str) -> Result { @@ -241,6 +272,7 @@ fn is_selector_kind(prefix: &str) -> bool { TAG_KIND, PATH_KIND, TEST_KIND, + JOB_KIND, ] .contains(&prefix) || UNMAPPED_KINDS.contains(&prefix) @@ -260,6 +292,7 @@ pub(crate) fn matched(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved Ok(keys) } PATH_KIND => folder(graph, value), + JOB_KIND => job(graph, value), TEST_KIND => Err(error( UNIT_TEST_CODE, format!( @@ -273,7 +306,17 @@ pub(crate) fn matched(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved return Ok(keys); } let pattern: bool = is_pattern(value); - let names: Vec<&str> = graph.names.iter().map(|(name, _)| name.as_str()).collect(); + let names: Vec<&str> = graph + .names + .iter() + .map(|(name, _)| name.as_str()) + .chain( + graph + .jobs + .iter() + .flat_map(|jobs| jobs.keys().map(String::as_str)), + ) + .collect(); Err(SelectorError { code: UNKNOWN_NAME_CODE, message: format!( @@ -309,6 +352,12 @@ fn lookup(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved { }; let kind_matches = |key: &ObjectKey| resource.is_none_or(|resource| key.0 == resource); if !is_pattern(value) { + if resource.is_none() + && graph.key_named(value).is_none() + && let Some(Some(keys)) = graph.jobs.as_ref().map(|jobs| jobs.get(value)) + { + return keys.clone(); + } return Ok(graph .key_named(value) .filter(|key| kind_matches(key)) @@ -324,6 +373,40 @@ fn lookup(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved { .collect()) } +/// The nodes the jobs and job groups `value` names or matches select. +fn job(graph: &ProjectGraph, value: &str) -> Resolved { + let Some(jobs) = &graph.jobs else { + return Err(error( + UNMAPPED_KIND_CODE, + format!( + "selector '{JOB_KIND}{KIND_SEPARATOR}{value}' selects a job; jobs cannot be selected here" + ), + )); + }; + let pattern: bool = is_pattern(value); + let mut keys: Keys = Keys::new(); + let mut found: bool = false; + for (name, selected) in jobs { + if name == value || (pattern && glob_matches(value, name)) { + found = true; + keys.extend(selected.clone()?); + } + } + if found { + return Ok(keys); + } + Err(SelectorError { + code: UNKNOWN_NAME_CODE, + message: format!( + "unknown job {} '{value}'", + if pattern { "pattern" } else { "name" } + ), + help: (!pattern) + .then(|| suggestion(value, jobs.keys().map(String::as_str).collect())) + .flatten(), + }) +} + fn folder(graph: &ProjectGraph, value: &str) -> Resolved { let folder: String = value.replace('\\', "/").trim_matches('/').to_owned(); let selector: &str = if folder == MODEL_ROOT { diff --git a/crates/sqlbuild-analysis/src/graph/constants.rs b/crates/sqlbuild-analysis/src/graph/constants.rs index 0d8ef8279d..19d224cb54 100644 --- a/crates/sqlbuild-analysis/src/graph/constants.rs +++ b/crates/sqlbuild-analysis/src/graph/constants.rs @@ -18,6 +18,7 @@ pub(crate) const SQL_FILE_SUFFIX: &str = ".sql"; pub(crate) const TAG_KIND: &str = "tag"; pub(crate) const PATH_KIND: &str = "path"; pub(crate) const TEST_KIND: &str = "test"; +pub(crate) const JOB_KIND: &str = "job"; /// Selector kinds Python parses but `sqb compile` cannot map to a compiled resource. pub(crate) const UNMAPPED_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; pub(crate) const SUGGESTION_CUTOFF: f64 = 0.8; diff --git a/crates/sqlbuild-analysis/src/graph/main/attach_jobs.rs b/crates/sqlbuild-analysis/src/graph/main/attach_jobs.rs new file mode 100644 index 0000000000..dfb30c7526 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/attach_jobs.rs @@ -0,0 +1,8 @@ +use std::collections::BTreeMap; + +use crate::graph::models::{JobSelection, ProjectGraph}; + +/// Attach jobs and job groups so `job:` and a bare job name select what each job selects. +pub fn attach_jobs(graph: ProjectGraph, catalog: BTreeMap) -> ProjectGraph { + crate::graph::_helpers::jobs::attach(graph, catalog) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/mod.rs b/crates/sqlbuild-analysis/src/graph/main/mod.rs index 65c5eea9e7..ccd21dc172 100644 --- a/crates/sqlbuild-analysis/src/graph/main/mod.rs +++ b/crates/sqlbuild-analysis/src/graph/main/mod.rs @@ -1,3 +1,4 @@ +pub mod attach_jobs; pub mod build_project_graph; pub mod build_resources; pub mod downstream_edges; @@ -11,4 +12,5 @@ pub mod reachable_keys; pub mod resolve_selector_tokens; pub mod resolve_selectors; pub mod selector_name_help; +pub mod selectors_need_jobs; pub mod unique_id_edges; diff --git a/crates/sqlbuild-analysis/src/graph/main/selectors_need_jobs.rs b/crates/sqlbuild-analysis/src/graph/main/selectors_need_jobs.rs new file mode 100644 index 0000000000..de8a837263 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/selectors_need_jobs.rs @@ -0,0 +1,10 @@ +use crate::graph::models::ProjectGraph; + +/// Whether `selectors` name a job, either by `job:` or as a bare name the graph and `also_known` lack. +pub fn selectors_need_jobs( + graph: &ProjectGraph, + selectors: &[String], + also_known: &[&str], +) -> bool { + crate::graph::_helpers::selectors::needs_jobs(graph, selectors, also_known) +} diff --git a/crates/sqlbuild-analysis/src/graph/models.rs b/crates/sqlbuild-analysis/src/graph/models.rs index 61ffc53442..a4247f5b7c 100644 --- a/crates/sqlbuild-analysis/src/graph/models.rs +++ b/crates/sqlbuild-analysis/src/graph/models.rs @@ -1,8 +1,9 @@ //! The project graph, the resources it is built from and selector failures. -use std::collections::HashMap; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; /// Dependency lists indexed without losing project insertion order. #[derive(Debug, Default)] @@ -37,6 +38,24 @@ pub struct ProjectGraph { pub(crate) upstream_index: HashMap, pub(crate) downstream_index: HashMap, pub(crate) name_index: HashMap, + /// Each job and job group's resolved selection, or why it fails; `None` until a command attaches jobs. + pub jobs: Option, SelectorError>>>, +} + +/// What one job or job group selects: its own selector tokens, its member jobs, or resolved keys. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum JobSelection { + Job { + select: Vec, + exclude: Vec, + }, + Group { + members: Vec, + }, + /// A selection already resolved, such as a published job's. + Resolved { + keys: Vec, + }, } /// One parsed selector token: `kind:value` (kind `name` when bare) or `start~end`. diff --git a/crates/sqlbuild-analysis/src/graph/tests.rs b/crates/sqlbuild-analysis/src/graph/tests.rs index 26c1d3f80a..2629f1cacf 100644 --- a/crates/sqlbuild-analysis/src/graph/tests.rs +++ b/crates/sqlbuild-analysis/src/graph/tests.rs @@ -2,6 +2,8 @@ mod graph; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_job_selectors.rs"] +mod job_selectors; #[path = "tests/test_selector_scenarios.rs"] mod selector_scenarios; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/graph/tests/helpers.rs b/crates/sqlbuild-analysis/src/graph/tests/helpers.rs index ec596f7f1a..3174ac179a 100644 --- a/crates/sqlbuild-analysis/src/graph/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/graph/tests/helpers.rs @@ -1,10 +1,12 @@ -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use crate::assembly::project::types::ObjectKey; use crate::graph::errors::SelectorError; +use crate::graph::main::attach_jobs::attach_jobs; use crate::graph::main::build_project_graph::build_project_graph; use crate::graph::main::resolve_selectors::resolve_selectors; -use crate::graph::models::{GraphResource, ParsedSelector, ProjectGraph}; +use crate::graph::main::selectors_need_jobs::selectors_need_jobs; +use crate::graph::models::{GraphResource, JobSelection, ParsedSelector, ProjectGraph}; use crate::graph::tests::test_types::SelectorFailure; type OwnedFailure = (String, String, Option); @@ -225,3 +227,48 @@ pub(super) fn path(start: &str, end: &str, upstream: bool, downstream: bool) -> downstream, } } + +/// Jobs over the orders project: valid jobs, a group, and jobs whose own selection fails. +fn orders_jobs() -> BTreeMap { + let job = |select: &[&str], exclude: &[&str]| JobSelection::Job { + select: select.iter().map(|value| (*value).to_owned()).collect(), + exclude: exclude.iter().map(|value| (*value).to_owned()).collect(), + }; + let group = |members: &[&str]| JobSelection::Group { + members: members.iter().map(|value| (*value).to_owned()).collect(), + }; + BTreeMap::from([ + ("nightly".to_owned(), job(&["orders"], &[])), + ("crm".to_owned(), job(&["customers"], &[])), + ("marts".to_owned(), job(&["orders+"], &[])), + ( + "marts_without_items".to_owned(), + job(&["orders+"], &["order_items"]), + ), + ("daily_group".to_owned(), group(&["nightly", "crm"])), + ("broken".to_owned(), job(&["missing"], &[])), + ("nested".to_owned(), job(&["job:nightly"], &[])), + ("ghost_group".to_owned(), group(&["nightly", "ghost"])), + ]) +} + +/// The names `select` minus `exclude` resolves to in the orders project, with or without its jobs. +pub(super) fn job_selection( + attached: bool, + select: &[&str], + exclude: &[&str], +) -> Result, OwnedFailure> { + let mut graph: ProjectGraph = attach_jobs( + orders_graph(), + orders_jobs().into_iter().filter(|_| attached).collect(), + ); + graph.jobs = graph.jobs.filter(|_| attached); + resolved_in(&graph, select, exclude) + .map(|keys| keys.into_iter().map(|(_, name)| name).collect()) +} + +/// Whether `selectors` need jobs on the orders project, `also_known` naming other node names. +pub(super) fn needs_jobs(selectors: &[&str], also_known: &[&str]) -> bool { + let selectors: Vec = selectors.iter().map(|value| (*value).to_owned()).collect(); + selectors_need_jobs(&orders_graph(), &selectors, also_known) +} diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_job_selectors.rs b/crates/sqlbuild-analysis/src/graph/tests/test_job_selectors.rs new file mode 100644 index 0000000000..d196f7ee4f --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/tests/test_job_selectors.rs @@ -0,0 +1,212 @@ +use crate::graph::tests::helpers::{failure, job_selection, names, needs_jobs}; +use crate::graph::tests::test_types::{ + JobSelectorErrorTestCase, JobSelectorTestCase, NeedsJobsTestCase, +}; + +#[test] +fn given_attached_jobs_when_resolving_then_jobs_select_their_own_nodes() { + let test_cases = [ + JobSelectorTestCase { + description: "a job selects its selection and the functions it needs", + select: &["job:nightly"], + exclude: &[], + expected_names: &["orders", "tax"], + }, + JobSelectorTestCase { + description: "a bare job name needs no prefix", + select: &["nightly"], + exclude: &[], + expected_names: &["orders", "tax"], + }, + JobSelectorTestCase { + description: "a job's own exclusion holds", + select: &["job:marts_without_items"], + exclude: &[], + expected_names: &["customer_orders", "orders", "tax"], + }, + JobSelectorTestCase { + description: "a group selects the union of its jobs", + select: &["job:daily_group"], + exclude: &[], + expected_names: &["customers", "orders", "tax"], + }, + JobSelectorTestCase { + description: "a job intersects with other selectors", + select: &["job:marts,tag:finance"], + exclude: &[], + expected_names: &["order_items", "tax"], + }, + JobSelectorTestCase { + description: "a command exclusion narrows a job", + select: &["job:marts"], + exclude: &["order_items"], + expected_names: &["customer_orders", "orders", "tax"], + }, + JobSelectorTestCase { + description: "a job can be excluded", + select: &["job:marts"], + exclude: &["job:nightly"], + expected_names: &["customer_orders", "order_items", "tax"], + }, + JobSelectorTestCase { + description: "expansion applies to the job's nodes", + select: &["+job:crm+"], + exclude: &[], + expected_names: &["customer_orders", "customers", "tax"], + }, + JobSelectorTestCase { + description: "a job pattern matches job names", + select: &["job:cr?"], + exclude: &[], + expected_names: &["customers"], + }, + ]; + for test_case in test_cases { + assert_eq!( + job_selection(true, test_case.select, test_case.exclude), + Ok(names(test_case.expected_names)), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_job_selectors_when_unresolvable_then_fails_with_message() { + let test_cases = [ + JobSelectorErrorTestCase { + description: "jobs not loaded", + attached: false, + select: &["job:nightly"], + expected_error: ( + "S010", + "selector 'job:nightly' selects a job; jobs cannot be selected here", + None, + ), + }, + JobSelectorErrorTestCase { + description: "unknown job suggests a job", + attached: true, + select: &["job:nightyl"], + expected_error: ( + "S007", + "unknown job name 'nightyl'", + Some("did you mean 'nightly'?"), + ), + }, + JobSelectorErrorTestCase { + description: "unknown bare name suggests a job", + attached: true, + select: &["nightyl"], + expected_error: ( + "S007", + "unknown selector name 'nightyl'", + Some("did you mean 'nightly'?"), + ), + }, + JobSelectorErrorTestCase { + description: "a job's own selection failure surfaces", + attached: true, + select: &["job:broken"], + expected_error: ("S007", "unknown selector name 'missing'", None), + }, + JobSelectorErrorTestCase { + description: "a job cannot select another job", + attached: true, + select: &["job:nested"], + expected_error: ( + "S010", + "selector 'job:nightly' selects a job; jobs cannot be selected here", + None, + ), + }, + JobSelectorErrorTestCase { + description: "a job cannot end a path selector", + attached: true, + select: &["orders~nightly"], + expected_error: ( + "S003", + "path selector endpoint 'nightly' is a job; path selectors join two nodes", + None, + ), + }, + JobSelectorErrorTestCase { + description: "a job: prefix cannot start a path selector", + attached: false, + select: &["job:nightly~orders"], + expected_error: ( + "S003", + "path selector endpoint 'job:nightly' is a job; path selectors join two nodes", + None, + ), + }, + JobSelectorErrorTestCase { + description: "a group with an unknown member", + attached: true, + select: &["job:ghost_group"], + expected_error: ( + "S007", + "job group 'ghost_group' names unknown job 'ghost'", + None, + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + job_selection(test_case.attached, test_case.select, &[]), + Err(failure(test_case.expected_error)), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selectors_when_scanning_then_only_job_references_need_jobs() { + let test_cases = [ + NeedsJobsTestCase { + description: "job prefix", + selectors: &["job:nightly"], + also_known: &[], + expected_needs_jobs: true, + }, + NeedsJobsTestCase { + description: "graph name", + selectors: &["+orders+"], + also_known: &[], + expected_needs_jobs: false, + }, + NeedsJobsTestCase { + description: "bare name outside the graph", + selectors: &["nightly"], + also_known: &[], + expected_needs_jobs: true, + }, + NeedsJobsTestCase { + description: "job inside an intersection of a later token", + selectors: &["orders", "tag:daily,job:nightly"], + also_known: &[], + expected_needs_jobs: true, + }, + NeedsJobsTestCase { + description: "name another selector knows", + selectors: &["load_orders"], + also_known: &["load_orders"], + expected_needs_jobs: false, + }, + NeedsJobsTestCase { + description: "patterns, paths and folders", + selectors: &["night*", "orders~nightly", "path:models/mart", "tag:daily"], + also_known: &[], + expected_needs_jobs: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + needs_jobs(test_case.selectors, test_case.also_known), + test_case.expected_needs_jobs, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_types.rs b/crates/sqlbuild-analysis/src/graph/tests/test_types.rs index 2bb69273a2..851e746137 100644 --- a/crates/sqlbuild-analysis/src/graph/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/graph/tests/test_types.rs @@ -52,3 +52,24 @@ pub(super) struct ProjectSelectorErrorTestCase { pub(super) select: &'static [&'static str], pub(super) expected_error: SelectorFailure, } + +pub(super) struct JobSelectorTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) exclude: &'static [&'static str], + pub(super) expected_names: &'static [&'static str], +} + +pub(super) struct JobSelectorErrorTestCase { + pub(super) description: &'static str, + pub(super) attached: bool, + pub(super) select: &'static [&'static str], + pub(super) expected_error: SelectorFailure, +} + +pub(super) struct NeedsJobsTestCase { + pub(super) description: &'static str, + pub(super) selectors: &'static [&'static str], + pub(super) also_known: &'static [&'static str], + pub(super) expected_needs_jobs: bool, +} diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs index 34f8bf1c3a..e2390a571b 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs @@ -11,10 +11,11 @@ use crate::graph::errors::SelectorError; use crate::graph::main::parse_selector::parse_selector; use crate::graph::models::{EdgeDirection, ParsedSelector, ProjectGraph}; use crate::python_selection::constants::{ - ASSET_KIND, CHECK_KIND, INTERSECTION_SEPARATOR, LOADER_KIND, NAME_KIND, PATH_BETWEEN_CODE, - PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PATTERN_CHARACTERS, SQL_MODEL_PATH_ROOT, TAG_KIND, - TASK_KIND, TEST_KIND, UNIT_TEST_CODE, UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, - UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, + ASSET_KIND, CHECK_KIND, INTERSECTION_SEPARATOR, JOB_KIND, LOADER_KIND, NAME_KIND, + PATH_BETWEEN_CODE, PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PATTERN_CHARACTERS, + SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, UNIT_TEST_CODE, + UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, + UNMAPPED_KIND_CODE, }; use crate::python_selection::models::{NodeIndex, PythonSelectionGraph, PythonSelectionNode}; use sqlbuild_core::constants::PYTHON_NODE_ROOT as PYTHON_NODE_PATH_ROOT; @@ -282,3 +283,10 @@ pub(crate) fn sql_names(graph: &ProjectGraph) -> HashMap<&str, &ObjectKey> { .map(|(name, key)| (name.as_str(), key)) .collect() } + +/// Whether a part of `token` is a `job:` selector; jobs select only SQL resources. +pub(crate) fn names_job(token: &str) -> bool { + token.split(INTERSECTION_SEPARATOR).any(|part| { + matches!(parse_selector(part), Ok(ParsedSelector::Kind { kind, .. }) if kind == JOB_KIND) + }) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 523963f25e..b98e48dca6 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -20,10 +20,10 @@ use crate::python_selection::_helpers::validation::{ source_loaders, terminal_loaders, validate_boundaries, validate_selected_dependencies, }; use crate::python_selection::constants::{ - ASSET_KIND, CHECK_KIND, EXPANSION_MARKER, INTERSECTION_SEPARATOR, LOADER_KIND, NAME_KIND, - PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PLANNER_DEFAULT_CODE, SEED_KIND, SOURCE_KIND, - SOURCE_RESOURCE, SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, UNKNOWN_NAME_CODE, - UNKNOWN_TAG_CODE, + ASSET_KIND, CHECK_KIND, EXPANSION_MARKER, INTERSECTION_SEPARATOR, JOB_KIND, LOADER_KIND, + NAME_KIND, PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PLANNER_DEFAULT_CODE, SEED_KIND, + SOURCE_KIND, SOURCE_RESOURCE, SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, + UNKNOWN_NAME_CODE, UNKNOWN_TAG_CODE, }; use crate::python_selection::models::{Atom, PythonSelectionNode, RunSelection, SelectionContext}; use sqlbuild_core::constants::PYTHON_NODE_ROOT as PYTHON_NODE_PATH_ROOT; @@ -258,7 +258,7 @@ fn single_side(context: &SelectionContext<'_>, raw: &str) -> Resolved { let atoms: Atoms = match kind.as_str() { TEST_KIND => return Err(unit_test_error(raw)), PATH_KIND => resolve_path(context, raw, &value)?, - SEED_KIND | SOURCE_KIND => resolve_sql(context, raw)?, + SEED_KIND | SOURCE_KIND | JOB_KIND => resolve_sql(context, raw)?, TASK_KIND | ASSET_KIND | LOADER_KIND | CHECK_KIND => { return resolve_typed(context, raw, (&kind, &value)); } @@ -357,6 +357,15 @@ fn both_error(name: &str) -> SelectorError { ) } +fn is_job(context: &SelectionContext<'_>, value: &str) -> bool { + context + .sql + .graph + .jobs + .as_ref() + .is_some_and(|jobs| jobs.contains_key(value)) +} + fn unknown_name(value: &str) -> SelectorError { error( UNKNOWN_NAME_CODE, @@ -374,6 +383,7 @@ fn resolve_name(context: &SelectionContext<'_>, raw: &str, value: &str) -> Resol (true, true) => Err(both_error(value)), (true, false) => resolve_sql(context, raw), (false, true) => resolve_python(context, raw), + (false, false) if is_job(context, value) => resolve_sql(context, raw), (false, false) => Err(unknown_name(value)), }; } diff --git a/crates/sqlbuild-analysis/src/python_selection/constants.rs b/crates/sqlbuild-analysis/src/python_selection/constants.rs index 202a3b2169..a563ace884 100644 --- a/crates/sqlbuild-analysis/src/python_selection/constants.rs +++ b/crates/sqlbuild-analysis/src/python_selection/constants.rs @@ -8,6 +8,7 @@ pub(crate) const NAME_KIND: &str = "name"; pub(crate) const TAG_KIND: &str = "tag"; pub(crate) const PATH_KIND: &str = "path"; pub(crate) const TEST_KIND: &str = "test"; +pub(crate) const JOB_KIND: &str = "job"; pub(crate) const SEED_KIND: &str = "seed"; pub(crate) const SOURCE_KIND: &str = "source"; pub(crate) const MODEL_RESOURCE: &str = "model"; diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_node_excludes.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_node_excludes.rs index 02d13d8d01..de5f26d169 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_node_excludes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_node_excludes.rs @@ -1,5 +1,5 @@ use crate::graph::errors::SelectorError; -use crate::python_selection::_helpers::nodes::select_nodes; +use crate::python_selection::_helpers::nodes::{names_job, select_nodes}; use crate::python_selection::constants::IGNORED_EXCLUDE_CODES; use crate::python_selection::models::{NodeIndex, PythonSelectionGraph}; @@ -11,6 +11,9 @@ pub fn resolve_python_node_excludes( let index: NodeIndex<'_> = NodeIndex::new(graph); let mut excluded: Vec = Vec::new(); for token in exclude.iter().flat_map(|group| group.split_whitespace()) { + if names_job(token) { + continue; + } match select_nodes(&index, &[token.to_owned()], &[]) { Ok(names) => excluded.extend(names), Err(error) if IGNORED_EXCLUDE_CODES.contains(&error.code) => {} diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index bd4293afd5..e6208078b5 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -3,12 +3,14 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::{Value as Json, json}; +use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; -use sqlbuild_analysis::graph::models::GraphResource; +use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; +use sqlbuild_analysis::graph::models::{GraphResource, JobSelection}; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use sqlbuild_orchestration::runs::models::{CommandPlan, PlannerResource}; -use sqlbuild_orchestration::runs::types::CommandPlanner; +use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs}; use crate::adhoc::constants::{ ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, COMMAND_DEST, FRESHNESS_VERB, INTEGER_CONVERTERS, @@ -27,7 +29,7 @@ impl CommandPlanner for CliPlanner { fn plan( &self, command: &[String], - resources: &[PlannerResource], + publication: (&[PlannerResource], &PublishedJobs), ) -> Result { let verb = command .first() @@ -61,7 +63,7 @@ impl CommandPlanner for CliPlanner { let exclude: Vec = selectors.remove("exclude").unwrap_or_default(); let selection: BTreeSet = match verb.as_str() { SEED_VERB => { - let seeds: BTreeSet = resolve(resources, &select, &exclude)? + let seeds: BTreeSet = resolve(publication, &select, &exclude)? .into_iter() .filter(|key| key.starts_with("seed:")) .collect(); @@ -70,8 +72,8 @@ impl CommandPlanner for CliPlanner { } seeds } - FRESHNESS_VERB => freshness_sources(resources, &select, &exclude)?, - _ => resolve(resources, &select, &exclude)?, + FRESHNESS_VERB => freshness_sources(publication, &select, &exclude)?, + _ => resolve(publication, &select, &exclude)?, }; Ok(CommandPlan { verb: verb.clone(), @@ -177,11 +179,11 @@ fn flag_of(dest: &str) -> String { /// Exact `kind:name` keys, as `sqb build` would select them from this publication. fn resolve( - resources: &[PlannerResource], + publication: (&[PlannerResource], &PublishedJobs), select: &[String], exclude: &[String], ) -> Result, String> { - let selected: BTreeSet = resolved(resources, select, exclude)?; + let selected: BTreeSet = resolved(publication, select, exclude)?; if selected.is_empty() { return Err("the selectors match no nodes in the current publication".into()); } @@ -189,12 +191,16 @@ fn resolve( } fn resolved( - resources: &[PlannerResource], + (resources, jobs): (&[PlannerResource], &PublishedJobs), select: &[String], exclude: &[String], ) -> Result, String> { let graph_resources: Vec = resources.iter().filter_map(graph_resource).collect(); - let graph = build_project_graph(&graph_resources); + let mut graph = build_project_graph(&graph_resources); + let selectors: Vec = select.iter().chain(exclude).cloned().collect(); + if selectors_need_jobs(&graph, &selectors, &[]) { + graph = attach_jobs(graph, published_jobs(jobs)); + } Ok(resolve_selectors(&graph, select, exclude) .map_err(|error| format!("{}: {}", error.code, error.message))? .into_iter() @@ -202,18 +208,28 @@ fn resolved( .collect()) } +/// Published jobs as already-resolved graph selections. +fn published_jobs(jobs: &PublishedJobs) -> BTreeMap { + let mut selections: BTreeMap = BTreeMap::new(); + for (name, keys) in jobs { + let keys: Vec<(String, String)> = keys.iter().filter_map(|key| object_key(key)).collect(); + selections.insert(name.clone(), JobSelection::Resolved { keys }); + } + selections +} + /// Sources `sqb freshness` observes: selected or upstream sources, less those the exclusions reach. fn freshness_sources( - resources: &[PlannerResource], + publication: (&[PlannerResource], &PublishedJobs), select: &[String], exclude: &[String], ) -> Result, String> { let selected: BTreeSet = - upstream_sources(resources, &resolved(resources, select, &[])?); + upstream_sources(publication.0, &resolved(publication, select, &[])?); let excluded: BTreeSet = if exclude.is_empty() { BTreeSet::new() } else { - upstream_sources(resources, &resolved(resources, exclude, &[])?) + upstream_sources(publication.0, &resolved(publication, exclude, &[])?) }; let sources: BTreeSet = selected.difference(&excluded).cloned().collect(); if sources.is_empty() { diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index c4d2fd6386..370c7bd492 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -1,6 +1,9 @@ //! The planner parses with the CLI's own table, refuses every flag outside the allowlist and resolves exact keys. +use std::collections::{BTreeMap, BTreeSet}; + use serde_json::json; +use sqlbuild_orchestration::runs::types::PublishedJobs; use crate::adhoc::main::adhoc_planner::adhoc_planner; use crate::adhoc::tests::helpers::{owned_plan, resource}; @@ -26,7 +29,38 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select resource("source:shop.raw_orders", &[], &[], None), resource("seed:currencies", &[], &[], None), ]; + let jobs: PublishedJobs = BTreeMap::from([ + ( + "nightly".to_owned(), + BTreeSet::from(["model:orders".to_owned(), "model:order_totals".to_owned()]), + ), + ( + "daily_group".to_owned(), + BTreeSet::from(["model:customers".to_owned(), "model:orders".to_owned()]), + ), + ]); let test_cases = [ + AdhocPlanTestCase { + description: "a published job selects its published nodes", + command: &[ + "build", + "--select", + "job:nightly", + "--exclude", + "order_totals", + ], + expected_plan: Ok((&["model:orders"], json!({}))), + }, + AdhocPlanTestCase { + description: "a bare published group name needs no prefix", + command: &["build", "--select", "daily_group"], + expected_plan: Ok((&["model:customers", "model:orders"], json!({}))), + }, + AdhocPlanTestCase { + description: "an unpublished job is refused", + command: &["build", "--select", "job:weekly"], + expected_plan: Err("unknown job name 'weekly'"), + }, AdhocPlanTestCase { description: "upstream selector reaches the source and seed it reads", command: &["build", "--select", "+orders"], @@ -217,7 +251,7 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select .map(|token| (*token).to_owned()) .collect(); let actual = planner - .plan(&command, &resources) + .plan(&command, (&resources, &jobs)) .map(|plan| { ( plan.selection.into_iter().collect::>(), diff --git a/crates/sqlbuild-cli/src/argv/_helpers/table.rs b/crates/sqlbuild-cli/src/argv/_helpers/table.rs index 12bfc5b45b..c094f1210f 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/table.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/table.rs @@ -23,6 +23,17 @@ pub(crate) fn root_parser() -> &'static ParserSpec { }) } +/// One `sqb` subcommand's parser, such as `build` or `plan`. +pub(crate) fn command_parser(command: &str) -> Result<&'static ParserSpec, String> { + root_parser() + .actions + .iter() + .flat_map(|action| &action.parsers) + .find(|(name, _)| name == command) + .map(|(_, parser)| parser) + .ok_or_else(|| format!("{command} command is unavailable")) +} + /// The `sqb dbt ` execution parser with the command named in its texts. pub(crate) fn dbt_execution_parser(command: &str) -> ParserSpec { let mut parser: ParserSpec = DBT_EXECUTION diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index c1810fa621..c433d00395 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -630,6 +630,24 @@ "help": "run UUID; reuse it with identical arguments after an uncertain result", "group": 1 }, + { + "kind": "extend", + "options": ["--select", "-s"], + "dest": "admin_select", + "nargs": "+", + "default": [], + "help": "run only the job's nodes these selectors also select", + "group": 1 + }, + { + "kind": "extend", + "options": ["--exclude"], + "dest": "admin_exclude", + "nargs": "+", + "default": [], + "help": "leave the job's nodes these selectors select out of the run", + "group": 1 + }, { "kind": "store", "options": ["--start-cursor-ts"], diff --git a/crates/sqlbuild-cli/src/argv/main/accepted_settings_arguments.rs b/crates/sqlbuild-cli/src/argv/main/accepted_settings_arguments.rs new file mode 100644 index 0000000000..3128f38f7b --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/accepted_settings_arguments.rs @@ -0,0 +1,26 @@ +//! Turn frozen settings into one command's flags, leaving out settings that command does not define. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value as Json; + +use crate::argv::_helpers::table::command_parser; +use crate::argv::main::command_settings_arguments::command_settings_arguments; + +/// The flags `command` takes from `settings`; a setting it lacks is dropped rather than refused. +pub fn accepted_settings_arguments( + command: &str, + settings: &BTreeMap, +) -> Result, String> { + let defined: BTreeSet<&str> = command_parser(command)? + .actions + .iter() + .filter_map(|action| action.dest.as_deref()) + .collect(); + let accepted: BTreeMap = settings + .iter() + .filter(|(name, _)| defined.contains(name.as_str())) + .map(|(name, value)| (name.clone(), value.clone())) + .collect(); + command_settings_arguments(command, &accepted) +} diff --git a/crates/sqlbuild-cli/src/argv/main/command_settings_arguments.rs b/crates/sqlbuild-cli/src/argv/main/command_settings_arguments.rs index 5087b6ff53..02c148f1d5 100644 --- a/crates/sqlbuild-cli/src/argv/main/command_settings_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/main/command_settings_arguments.rs @@ -5,20 +5,14 @@ use std::collections::BTreeMap; use serde_json::Value as Json; use crate::argv::_helpers::job_settings::{setting_argument, validate_mutex}; -use crate::argv::_helpers::table::root_parser; +use crate::argv::_helpers::table::command_parser; /// Produce argument tokens for `command`, never shell text; a setting the command lacks is refused. pub fn command_settings_arguments( command: &str, settings: &BTreeMap, ) -> Result, String> { - let parser = root_parser() - .actions - .iter() - .flat_map(|action| &action.parsers) - .find(|(name, _)| name == command) - .map(|(_, parser)| parser) - .ok_or_else(|| format!("{command} command is unavailable"))?; + let parser = command_parser(command)?; let mut arguments: Vec = Vec::new(); for (name, value) in settings { if let Some(argument) = setting_argument(parser, name, value)? { diff --git a/crates/sqlbuild-cli/src/argv/main/mod.rs b/crates/sqlbuild-cli/src/argv/main/mod.rs index 10bbe8475b..10cdc0c9bf 100644 --- a/crates/sqlbuild-cli/src/argv/main/mod.rs +++ b/crates/sqlbuild-cli/src/argv/main/mod.rs @@ -1,3 +1,4 @@ +pub mod accepted_settings_arguments; pub mod cli_namespace_defaults; pub mod command_settings_arguments; pub mod job_settings_arguments; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index 35d9b69dfd..75fbb92064 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; use serde_json::json; -use sqlbuild_orchestration::api::models::{ApiAuthentication, HumanLogin}; +use sqlbuild_orchestration::api::models::{ApiAuthentication, HumanLogin, ServerFacts}; use sqlbuild_orchestration::credentials::models::{AuditRetention, CredentialScope}; use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::{LocationWork, LocationWorkRequest}; @@ -122,6 +122,7 @@ pub(crate) fn control_admin(namespace: &Namespace) -> Result Result Result< - ( - ApiAuthentication, - (StatisticsRetention, AuditRetention, SnapshotRetention), - ), - ControlError, -> { + standalone: bool, +) -> Result<(ApiAuthentication, ServerFacts), ControlError> { let statistics = match retention_setting(namespace, "statistics-retention-days")? { Some(days) => StatisticsRetention::days(days)?, None => StatisticsRetention::default(), @@ -284,7 +282,13 @@ pub(crate) fn control_api_settings( }; Ok(( control_authentication(namespace)?, - (statistics, audit, snapshots), + ServerFacts { + version: env!("CARGO_PKG_VERSION").into(), + scheduler: standalone || !flag(namespace, "control_no_scheduler"), + audit, + statistics, + snapshots, + }, )) } diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 0b9f3ae3e4..b7ea9fa268 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -13,7 +13,9 @@ use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::api::main::serve_api::serve_api; use crate::adhoc::main::adhoc_planner::adhoc_planner; -use sqlbuild_orchestration::api::models::{ApiAuthentication, DevelopmentScope, HumanLogin}; +use sqlbuild_orchestration::api::models::{ + ApiAuthentication, DevelopmentScope, HumanLogin, ServerFacts, +}; use sqlbuild_orchestration::credentials::models::AuditRetention; use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; @@ -112,7 +114,11 @@ pub(crate) async fn run_development_roles( (ApiAuthentication { login: HumanLogin::Development(scope), ..ApiAuthentication::default() - }, (Vec::new(), adhoc_planner()), Default::default()), + }, (Vec::new(), adhoc_planner()), ServerFacts { + version: env!("CARGO_PKG_VERSION").into(), + scheduler, + ..ServerFacts::default() + }), (move || { drop(write_output(&[(Stream::Stderr, &format!( "Development API ready: {address}\nDevelopment location: {location_id}\nAuthentication disabled on loopback; triggers {}\n", @@ -700,7 +706,7 @@ pub(crate) async fn check_location_connection( ))) } -fn location_child(options: &LocationServe, script: &Path) -> Command { +pub(crate) fn location_child(options: &LocationServe, script: &Path) -> Command { let mut command = Command::new(script); command.arg("--project-dir").arg(&options.project_dir); if terminal_columns().is_err() { diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs index 7b06b015ef..1aa95faf73 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs @@ -2,21 +2,19 @@ use std::path::Path; -use sqlbuild_config::effective::main::effective_target_name::effective_target_name; -use sqlbuild_config::effective::models::TargetChoice; -use sqlbuild_config::models::ConfigValue; -use sqlbuild_config::project::main::load_local_config::load_local_config; use sqlbuild_config::project::main::load_project_config::load_project_config; use sqlbuild_config::project::models::{Built, ConfigPaths}; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_orchestration::store::errors::ControlError; +use crate::native_commands::main::effective_target::effective_target; + use crate::native_commands::constants::{ CUSTOM_RULES_DIRECTORY, EVENT_EXPORTERS_DIRECTORY, LEGACY_LOCAL_CONFIG_FILENAME, LEGACY_PROJECT_CONFIG_FILENAME, LIFECYCLE_SINK_PATH, LOCAL_CONFIG_FILENAME, SINKS_DIRECTORY, }; -fn object_field<'a>(built: &'a Built, name: &str) -> Option<&'a Built> { +pub(crate) fn object_field<'a>(built: &'a Built, name: &str) -> Option<&'a Built> { match built { Built::Object(_, fields) => fields .iter() @@ -39,7 +37,7 @@ fn names_lifecycle_sinks(config: &Built) -> bool { } /// The four configuration files of `project_dir`. -fn config_paths(project_dir: &Path) -> ConfigPaths { +pub(crate) fn config_paths(project_dir: &Path) -> ConfigPaths { ConfigPaths { project: project_dir.join(PROJECT_CONFIG_FILENAME), legacy_project: project_dir.join(LEGACY_PROJECT_CONFIG_FILENAME), @@ -75,35 +73,9 @@ pub(crate) fn development_target( project_dir: &Path, selected: Option, ) -> Result { - let paths = config_paths(project_dir); - let invalid = - || ControlError::Invalid("project configuration is invalid; run sqb debug".into()); - let project = load_project_config(&paths).map_err(|_| invalid())?; - let local = load_local_config(&paths).map_err(|_| invalid())?; - let text = |config: &Built, name: &str| match object_field(config, name) { - Some(Built::Value(ConfigValue::String(value))) => Some(value.clone()), - _ => None, - }; - let defined: Vec = [&project, &local] - .into_iter() - .filter_map(|config| object_field(config, "targets")) - .filter_map(|targets| match targets { - Built::Dict(targets) => Some(targets), - _ => None, + effective_target(project_dir, selected) + .map_err(ControlError::Invalid)? + .ok_or_else(|| { + ControlError::Invalid("sqb dev requires --target or a configured default target".into()) }) - .flatten() - .map(|(name, _)| name.clone()) - .collect(); - effective_target_name(TargetChoice { - selected, - local: text(&local, "target"), - default: text(&project, "default_target"), - defined, - }) - .map_err(|_| { - ControlError::Invalid("unknown development target; run sqb debug --target ".into()) - })? - .ok_or_else(|| { - ControlError::Invalid("sqb dev requires --target or a configured default target".into()) - }) } diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index a3c0d46a69..bd2eb9b498 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -103,3 +103,6 @@ pub(crate) const METADATA_REPLY_BYTES: u64 = 3 * sqlbuild_orchestration::api::constants::BODY_BYTES as u64; /// Bound piped terminal input independently of the credential owner's password policy. pub(crate) const PASSWORD_INPUT_BYTES: u64 = 4096; + +/// The command a location runs to answer a plan request. +pub(crate) const PLAN_COMMAND: &str = "plan"; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs new file mode 100644 index 0000000000..aee88ba2d4 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs @@ -0,0 +1,89 @@ +//! A requested plan runs as an ordinary `sqb plan --json` child on the server-frozen node keys. + +use std::collections::{BTreeMap, BTreeSet}; +use std::future::Future; +use std::path::Path; +use std::process::Stdio; + +use serde_json::{Value, json}; +use sqlbuild_orchestration::platform::constants::PLAN_DOCUMENT_BYTES; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::io::AsyncReadExt; +use tokio::process::Child; + +use crate::argv::main::accepted_settings_arguments::accepted_settings_arguments; +use crate::native_commands::_helpers::execution::location_child; +use crate::native_commands::constants::PLAN_COMMAND; +use crate::native_commands::models::LocationServe; + +/// The plan document, an `{"error": ...}` report when planning failed, or `None` when stopped first. +pub(super) async fn plan_location( + (options, script): (&LocationServe, &Path), + (selection, settings): (&BTreeSet, &BTreeMap), + stopped: impl Future>, +) -> Result, ControlError> { + let mut child = match plan_child((options, script), (selection, settings)) { + Ok(child) => child?, + Err(message) => return Ok(Some(failure(&message))), + }; + let stdout = child + .stdout + .take() + .ok_or_else(|| ControlError::Invalid("plan output is unavailable".into()))?; + let mut output: Vec = Vec::new(); + let limit = u64::try_from(PLAN_DOCUMENT_BYTES).unwrap_or(u64::MAX) + 1; + tokio::pin!(stopped); + let status = tokio::select! { + result = &mut stopped => { + child.kill().await?; + result?; + return Ok(None); + }, + result = async { + let mut reader = stdout.take(limit); + reader.read_to_end(&mut output).await?; + drop(reader); + child.wait().await + } => result?, + }; + if output.len() > PLAN_DOCUMENT_BYTES { + return Ok(Some(failure("the plan is too large to show"))); + } + Ok(Some(match serde_json::from_slice::(&output) { + Ok(document) if document.is_object() && matches!(status.code(), Some(0 | 1)) => document, + _ => failure("sqb plan did not produce a plan; see the location's log"), + })) +} + +/// The started child, or why the frozen request cannot become plan arguments. +fn plan_child( + (options, script): (&LocationServe, &Path), + (selection, settings): (&BTreeSet, &BTreeMap), +) -> Result, String> { + let flags = accepted_settings_arguments(PLAN_COMMAND, settings)?; + let mut command = location_child(options, script); + command + .args([ + PLAN_COMMAND, + "--target", + &options.target, + "--json", + "--no-color", + ]) + .args(flags); + for key in selection { + let (_, name) = key + .split_once(':') + .ok_or_else(|| "the plan request names an invalid node".to_owned())?; + command.args(["--select", name]); + } + Ok(command + .env_remove("SQLBUILD_LOCATION_TOKEN") + .env_remove("SQLBUILD_RUN_TOKEN") + .stdout(Stdio::piped()) + .spawn()) +} + +fn failure(message: &str) -> Value { + json!({"error": {"code": "plan_failed", "message": message, "help": null}}) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index abf575eb77..d67613c7a2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -16,6 +16,7 @@ use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::check_location_connection; use crate::native_commands::_helpers::paths::location_inputs; use crate::native_commands::_helpers::terminal::write_output; +use crate::native_commands::main::_location_plan::plan_location; use crate::native_commands::main::_location_sensor::evaluate_location_sensor; use crate::native_commands::models::{LocationServe, LocationWatch}; @@ -51,6 +52,7 @@ pub(super) async fn run_location_work( let message = match &claim.request.work { LocationWork::CheckConnection {} => "Checking requested location connection...\n", LocationWork::EvaluateSensor { .. } => "Evaluating requested custom sensor...\n", + LocationWork::Plan { .. } => "Planning requested command...\n", }; drop(write_output(&[(Stream::Stderr, message)])); let mut shutting_down = false; @@ -74,6 +76,13 @@ pub(super) async fn run_location_work( ) }) } + LocationWork::Plan { + selection, + settings, + .. + } => plan_location((options, script), (&selection, &settings), expires) + .await? + .map(|document| (LocationWorkOutcome::Plan { document }, Vec::new())), }; let Some((outcome, local_files)) = checked else { drop(write_output(&[( diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs index c61b8cb137..5eb26c8aeb 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_admin.rs @@ -1,5 +1,6 @@ //! Direct database ownership authorizes bootstrap, not an untrusted HTTP request or supplied role. +use crate::adhoc::main::adhoc_planner::adhoc_planner; use serde_json::{Value, json}; use sqlbuild_orchestration::authorization::main::bootstrap_user::bootstrap_user; use sqlbuild_orchestration::authorization::main::grant_admin::grant_admin; @@ -17,8 +18,10 @@ use sqlbuild_orchestration::platform::main::request_location_work::request_locat use sqlbuild_orchestration::runs::main::change_run::change_run; use sqlbuild_orchestration::runs::main::request_job_run::request_job_run; use sqlbuild_orchestration::runs::main::request_rerun::request_rerun; +use sqlbuild_orchestration::runs::models::JobRunRequest; use sqlbuild_orchestration::runs::models::{RunAuthority, RunChange, RunChangeRequest}; use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; use uuid::Uuid; use crate::argv::models::Namespace; @@ -73,10 +76,7 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result { - let applied = request_job_run(&store, &context, &request).await?; - Ok(json!({"run_id": request.run_id, "applied": applied})) - } + ControlAdmin::RequestJobRun(request) => queue_job_run(&store, &context, &request).await, ControlAdmin::Rerun(request) => { let applied = request_rerun(&store, (&context, None), &request).await?; Ok(json!({"run_id": request.run_id, "applied": applied})) @@ -131,3 +131,14 @@ pub(crate) async fn run_admin(namespace: &Namespace) -> Result Result { + let planner = adhoc_planner(); + let applied = request_job_run(store, (context, Some(planner.as_ref())), request).await?; + Ok(json!({"run_id": request.run_id, "applied": applied})) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs index 673283718f..60753b43f3 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_control.rs @@ -8,7 +8,7 @@ use tokio::sync::watch; use uuid::Uuid; use crate::argv::models::{Namespace, Stream}; -use crate::native_commands::_helpers::decision::{control_address, control_api_settings, flag}; +use crate::native_commands::_helpers::decision::{control_address, control_api_settings}; use crate::native_commands::_helpers::execution::{ open_control_store, open_control_workers, start_platform_roles, supervise_control_roles, }; @@ -27,7 +27,7 @@ pub(crate) async fn run_control( standalone: bool, ) -> Result<(), ControlError> { let address = control_address(namespace, standalone)?; - let (authentication, (statistics, audit, snapshots)) = control_api_settings(namespace)?; + let (authentication, facts) = control_api_settings(namespace, standalone)?; let store = open_control_store(namespace).await?; let (workers, handoff) = open_control_workers(namespace)?; let (stop, shutdown) = watch::channel(false); @@ -42,7 +42,7 @@ pub(crate) async fn run_control( tasks.spawn(serve_api( store.clone(), listener, - (authentication, handoff, statistics), + (authentication, handoff, facts.clone()), ( move || { drop(write_output(&[( @@ -56,10 +56,10 @@ pub(crate) async fn run_control( start_platform_roles( &mut tasks, (&store, &shutdown), - (audit, statistics, snapshots), + (facts.audit, facts.statistics, facts.snapshots), )?; } - if standalone || !flag(namespace, "control_no_scheduler") { + if facts.scheduler { drop(write_output(&[(Stream::Stderr, "Starting scheduler...\n")])); let scheduler_store = store.clone(); tasks.spawn(async move { diff --git a/crates/sqlbuild-cli/src/native_commands/main/effective_target.rs b/crates/sqlbuild-cli/src/native_commands/main/effective_target.rs new file mode 100644 index 0000000000..5794ca70e3 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/effective_target.rs @@ -0,0 +1,44 @@ +//! The command's target by the same selected, local, then project default precedence as compilation. + +use std::path::Path; + +use sqlbuild_config::effective::main::effective_target_name::effective_target_name; +use sqlbuild_config::effective::models::TargetChoice; +use sqlbuild_config::models::ConfigValue; +use sqlbuild_config::project::main::load_local_config::load_local_config; +use sqlbuild_config::project::main::load_project_config::load_project_config; +use sqlbuild_config::project::models::Built; + +use crate::native_commands::_helpers::project_facts::{config_paths, object_field}; + +/// The effective target of `project_dir` with `selected` as `--target`, `None` when nothing selects one. +pub fn effective_target( + project_dir: &Path, + selected: Option, +) -> Result, String> { + let paths = config_paths(project_dir); + let invalid = |_| "project configuration is invalid; run sqb debug".to_owned(); + let project = load_project_config(&paths).map_err(invalid)?; + let local = load_local_config(&paths).map_err(invalid)?; + let text = |config: &Built, name: &str| match object_field(config, name) { + Some(Built::Value(ConfigValue::String(value))) => Some(value.clone()), + _ => None, + }; + let defined: Vec = [&project, &local] + .into_iter() + .filter_map(|config| object_field(config, "targets")) + .filter_map(|targets| match targets { + Built::Dict(targets) => Some(targets), + _ => None, + }) + .flatten() + .map(|(name, _)| name.clone()) + .collect(); + effective_target_name(TargetChoice { + selected, + local: text(&local, "target"), + default: text(&project, "default_target"), + defined, + }) + .map_err(|_| "unknown target; run sqb debug --target ".to_owned()) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index c5fb741b92..d308f7d5b8 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1,6 +1,7 @@ pub(super) mod _cost_report_options; pub(super) mod _location_compiler; pub(super) mod _location_metadata; +pub(super) mod _location_plan; pub(super) mod _location_sensor; pub(super) mod _location_work; pub(super) mod _run_admin; @@ -8,6 +9,7 @@ pub(super) mod _run_control; pub(super) mod _run_cost; pub(super) mod _run_dev; pub(super) mod _run_location; +pub mod effective_target; pub(crate) mod parse_native_invocation; pub(crate) mod run_control_invocation; pub(crate) mod run_location_invocation; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index 929078663a..916a5810d0 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -69,6 +69,10 @@ pub(crate) fn person_routes() -> Router { "/api/admin/people", get(crate::api::_helpers::resources::administrators::people), ) + .route( + "/api/admin/server", + get(crate::api::_helpers::resources::administrators::server), + ) .route( "/api/admin/locations/{location_id}/targets/{target}/roles", get(crate::api::_helpers::resources::administrators::definitions), @@ -110,6 +114,14 @@ pub(crate) fn person_routes() -> Router { "/api/control/locations/{location_id}/targets/{target}/connection-tests/{request_id}", get(crate::api::_helpers::resources::person_location_work::result), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/plans", + post(crate::api::_helpers::resources::person_plans::request), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/plans/{request_id}", + get(crate::api::_helpers::resources::person_plans::read), + ) .route( "/api/control/locations/{location_id}/targets/{target}/runs", get(crate::api::_helpers::resources::person_operations::recent_runs), @@ -142,6 +154,10 @@ pub(crate) fn person_routes() -> Router { "/api/control/locations/{location_id}/targets/{target}/triggers/{trigger_id}/changes", get(crate::api::_helpers::resources::person_reads::trigger_changes), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/triggers/{trigger_id}/sensor", + get(crate::api::_helpers::resources::person_reads::sensor_state), + ) .route( "/api/control/locations/{location_id}/targets/{target}/snapshots", get(crate::api::_helpers::resources::person_reads::snapshots), @@ -154,6 +170,10 @@ pub(crate) fn person_routes() -> Router { "/api/control/platform/problems", get(crate::api::_helpers::resources::person_reads::problems), ) + .route( + "/api/control/platform/events", + get(crate::api::_helpers::resources::person_reads::platform_events), + ) .route( "/api/control/locations/{location_id}/targets/{target}/column-lineage", get(crate::api::_helpers::resources::person_operations::column_lineage), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs index 72c7bc9589..f06b608797 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs @@ -5,7 +5,9 @@ use axum::{Extension, Json}; use uuid::Uuid; use crate::api::errors::ApiError; -use crate::api::models::MutationResponse; +use crate::api::models::{ + LoginMode, MutationResponse, ServerFacts, ServerRetention, ServerSettings, +}; use crate::authorization::main::assign_builtin_role::assign_builtin_role; use crate::authorization::main::change_administrator::change_administrator; use crate::authorization::main::list_role_definitions::list_role_definitions; @@ -16,6 +18,11 @@ use crate::credentials::models::{ AdministratorChange, AuditContext, PersonSummary, RoleAssignmentRequest, RoleHoldings, RoleOverview, }; +use crate::platform::constants::{ + HEARTBEAT_RETENTION_DAYS, LOCATION_RETENTION_DAYS, PLATFORM_ALERT_RETENTION_DAYS, + SENSOR_EVALUATION_RETENTION_DAYS, +}; +use crate::platform::main::read_installation::read_installation; use crate::store::models::ControlStore; #[utoipa::path(post, path = "/api/admin/people/{principal_id}", operation_id = "changeAdministrator", @@ -87,3 +94,29 @@ pub(crate) async fn definitions( list_role_definitions(&store, (&context, principal), (location_id, &target)).await?, )) } + +#[utoipa::path(get, path = "/api/admin/server", operation_id = "readServerSettings", + responses((status = 200, body = ServerSettings), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn server( + (State(store), Extension(context)): (State, Extension), + (Extension(facts), Extension(login_mode)): (Extension, Extension), + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result, ApiError> { + let installation = read_installation(&store, (&context, principal)).await?; + Ok(Json(ServerSettings { + version: facts.version, + login_mode, + scheduler: facts.scheduler, + installation, + retention: ServerRetention { + audit_days: i64::from(facts.audit.kept_days()), + statistics_days: i64::from(facts.statistics.kept_days()), + snapshots_kept: i64::from(facts.snapshots.kept_count()), + platform_alert_days: PLATFORM_ALERT_RETENTION_DAYS, + heartbeat_days: HEARTBEAT_RETENTION_DAYS, + location_health_days: LOCATION_RETENTION_DAYS, + sensor_evaluation_days: SENSOR_EVALUATION_RETENTION_DAYS, + }, + })) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs index 9502df1756..544e4fc16c 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs @@ -17,11 +17,12 @@ use crate::api::_helpers::authentication::password_transport::session_credential use crate::api::errors::ApiError; use crate::api::models::{ChangeFeed, ChangeNotice, ChangeStream, ChangeSubscriber}; use crate::authorization::main::authorize_listing::authorize_listing; -use crate::authorization::main::person_lists_target::person_lists_target; +use crate::authorization::main::person_holds_target::person_holds_target; use crate::credentials::main::sessions::authenticate_session::authenticate_session; use crate::credentials::main::tokens::authenticate_token::authenticate_token; use crate::credentials::models::{ - AuditContext, AuthenticatedPerson, CredentialScope, DefaultRole, SessionAccess, TokenIdentity, + Action, AuditContext, AuthenticatedPerson, CredentialScope, DefaultRole, SessionAccess, + TokenIdentity, }; use crate::store::errors::ControlError; use crate::store::main::listen_changes::listen_changes; @@ -156,10 +157,10 @@ async fn still_authorized(state: &ChangeStream) -> Result { return Ok(false); } } - person_lists_target( + person_holds_target( &state.store, *principal_id, - (state.location_id, &state.target), + (Action::View, state.location_id, &state.target), *default_role, ) .await diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs index 0eddfb9c89..043cd0ad73 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod person_location_work; pub(crate) mod person_maintenance; pub(crate) mod person_node_sql; pub(crate) mod person_operations; +pub(crate) mod person_plans; pub(crate) mod person_reads; pub(crate) mod request_json; pub(crate) mod router; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index d510ca2834..97eb53cf64 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -7,7 +7,7 @@ use axum::{Extension, Json}; use uuid::Uuid; use crate::api::errors::ApiError; -use crate::api::models::{MutationResponse, ReadQuery}; +use crate::api::models::{MutationResponse, RunsQuery}; use crate::credentials::models::AuthenticatedPerson; use crate::credentials::models::{AuditContext, DefaultRole}; use crate::declarations::main::change_person_trigger::change_person_trigger; @@ -20,7 +20,8 @@ use crate::runs::models::{ AdhocCheck, AdhocCheckRequest, AdhocRunRequest, CancelRunRequest, JobRunRequest, PersonRunRequest, RerunRequest, RunRecord, }; -use crate::runs::types::CommandPlanner; +use crate::runs::types::{CommandPlanner, RunsListing}; +use crate::store::errors::ControlError; use crate::store::models::ControlStore; use crate::warehouse_metadata::main::read_person_catalog::read_person_catalog; use crate::warehouse_metadata::main::read_person_column_lineage::read_person_column_lineage; @@ -67,14 +68,17 @@ pub(crate) async fn statistics( (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn request_job( (State(store), Extension(context)): (State, Extension), - Extension(AuthenticatedPerson(principal, _)): Extension, + (Extension(AuthenticatedPerson(principal, _)), Extension(planner)): ( + Extension, + Extension>, + ), Json(request): Json, ) -> Result, ApiError> { Ok(Json(MutationResponse { applied: request_person_run( &store, (&context, principal), - PersonRunRequest::Job(&request), + PersonRunRequest::Job(planner.as_ref(), &request), ) .await?, })) @@ -168,20 +172,47 @@ pub(crate) async fn change_trigger( } #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/runs", operation_id = "listPersonRuns", - params(("location_id" = Uuid, Path), ("target" = String, Path), ("limit" = Option, Query)), - responses((status = 200, body = Vec), (status = 401, description = "Session rejected"), + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("limit" = Option, Query, description = "Page size; defaults to 100, capped at 1000"), + ("before_queued_at" = Option>, Query, description = "With before_run_id: continue after this run of the previous page"), + ("before_run_id" = Option, Query), + ("queued" = Option, Query, description = "List queued runs in claim order, each with its position and what it waits on")), + responses((status = 200, body = Vec, description = "Runs newest first, or the queue in claim order"), + (status = 400, description = "Only one of the before_* pair was given, or a cursor with queued"), + (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn recent_runs( (State(store), Extension(context)): (State, Extension), Extension(default_role): Extension, - (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), Extension(AuthenticatedPerson(principal, _)): Extension, ) -> Result>, ApiError> { + let limit = options.limit.unwrap_or(100); + let listing = match ( + options.queued, + options.before_queued_at, + options.before_run_id, + ) { + (false, Some(queued_at), Some(run_id)) => { + RunsListing::Newest(limit, Some((queued_at, run_id))) + } + (false, None, None) => RunsListing::Newest(limit, None), + (true, None, None) => RunsListing::Queue(limit), + (true, ..) => { + return Err(ControlError::Invalid("the queue is not paged".into()).into()); + } + _ => { + return Err(ControlError::Invalid( + "before_queued_at and before_run_id go together".into(), + ) + .into()); + } + }; Ok(Json( list_person_runs( &store, (&context, principal), - (location_id, &target, options.limit.unwrap_or(100)), + (location_id, &target, listing), default_role, ) .await?, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_plans.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_plans.rs new file mode 100644 index 0000000000..219cd6a4a4 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_plans.rs @@ -0,0 +1,63 @@ +//! A person plans a build command on the location; the plan never becomes a run. + +use std::sync::Arc; + +use axum::extract::{Path, State}; +use axum::{Extension, Json}; +use uuid::Uuid; + +use crate::api::errors::ApiError; +use crate::api::models::MutationResponse; +use crate::credentials::models::{AuditContext, AuthenticatedPerson}; +use crate::platform::main::read_person_plan::read_person_plan; +use crate::platform::models::{LocationWorkRecord, PlanRequest}; +use crate::runs::main::request_person_run::request_person_run; +use crate::runs::models::PersonRunRequest; +use crate::runs::types::CommandPlanner; +use crate::store::models::ControlStore; + +#[utoipa::path(post, path = "/api/control/locations/{location_id}/targets/{target}/plans", + operation_id = "requestPlan", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("x-sqb-csrf" = String, Header)), + request_body = PlanRequest, + responses((status = 200, body = MutationResponse), (status = 400, body = crate::api::models::ApiErrorBody), + (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn request( + (State(store), Extension(context)): (State, Extension), + Path((location_id, target)): Path<(Uuid, String)>, + (Extension(AuthenticatedPerson(principal, _)), Extension(planner)): ( + Extension, + Extension>, + ), + Json(request): Json, +) -> Result, ApiError> { + Ok(Json(MutationResponse { + applied: request_person_run( + &store, + (&context, principal), + PersonRunRequest::Plan(planner.as_ref(), (location_id, &target, &request)), + ) + .await?, + })) +} + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/plans/{request_id}", + operation_id = "readPlan", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("request_id" = Uuid, Path)), + responses((status = 200, body = Option), (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn read( + (State(store), Extension(context)): (State, Extension), + Path((location_id, target, request_id)): Path<(Uuid, String, Uuid)>, + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result>, ApiError> { + Ok(Json( + read_person_plan( + &store, + (&context, principal), + (location_id, &target, request_id), + ) + .await?, + )) +} diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs index 708bbf96b7..66d81a041a 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs @@ -6,7 +6,9 @@ use axum::{Extension, Json}; use uuid::Uuid; use crate::api::errors::ApiError; -use crate::api::models::{ReadQuery, RunEventsQuery, RunNodePath, StatementSqlQuery}; +use crate::api::models::{ + PlatformEventsQuery, ReadQuery, RunEventsQuery, RunNodePath, StatementSqlQuery, +}; use crate::credentials::models::{AuditContext, AuthenticatedPerson, DefaultRole}; use crate::declarations::main::list_person_jobs::list_person_jobs; use crate::declarations::main::list_person_trigger_changes::list_person_trigger_changes; @@ -15,9 +17,12 @@ use crate::declarations::main::read_person_job::read_person_job; use crate::declarations::models::{ JobDetailRecord, JobRecord, PersonTriggerChange, PersonTriggerRecord, }; +use crate::platform::main::list_person_platform_events::list_person_platform_events; use crate::platform::main::read_person_location_health::read_person_location_health; use crate::platform::main::read_person_problems::read_person_problems; -use crate::platform::models::{LocationHealthObservation, PlatformProblems}; +use crate::platform::models::{ + LocationHealthObservation, PersonPlatformEventPage, PlatformProblems, +}; use crate::runs::main::list_person_run_nodes::list_person_run_nodes; use crate::runs::models::{ NodeRunRecord, RunEventRecord, RunNodeFailure, RunNodeRecord, RunOutput, RunStatementRecord, @@ -25,6 +30,8 @@ use crate::runs::models::{ }; use crate::scheduler::main::list_person_ticks::list_person_ticks; use crate::scheduler::models::TickRecord; +use crate::sensors::main::read_person_sensor::read_person_sensor; +use crate::sensors::models::PersonSensorState; use crate::snapshots::main::list_person_snapshots::list_person_snapshots; use crate::snapshots::models::PublishedSnapshot; use crate::store::models::ControlStore; @@ -367,6 +374,37 @@ pub(crate) async fn trigger_changes( .ok_or_else(|| not_found("trigger not found")) } +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/triggers/{trigger_id}/sensor", operation_id = "readPersonSensorState", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("trigger_id" = Uuid, Path), + ("limit" = Option, Query, description = "Maximum recent checks and run requests; defaults to 100, capped at 1000")), + responses((status = 200, body = PersonSensorState, description = "The cursor, recent checks newest first and requested runs"), + (status = 401, description = "Session rejected"), + (status = 403, body = crate::api::models::ApiErrorBody), + (status = 404, description = "Sensor not found for a person who may view the whole target")), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn sensor_state( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target, trigger_id)), Query(options)): ( + Path<(Uuid, String, Uuid)>, + Query, + ), + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result, ApiError> { + read_person_sensor( + &store, + (&context, principal), + ( + (location_id, &target), + (trigger_id, options.limit.unwrap_or(100)), + ), + default_role, + ) + .await? + .map(Json) + .ok_or_else(|| not_found("sensor not found")) +} + #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/snapshots", operation_id = "listPersonSnapshots", params(("location_id" = Uuid, Path), ("target" = String, Path), ("limit" = Option, Query, description = "Maximum recent publications; defaults to 20, capped at 100")), @@ -425,6 +463,27 @@ pub(crate) async fn problems( )) } +#[utoipa::path(get, path = "/api/control/platform/events", operation_id = "listPersonPlatformEvents", + params(("before" = Option, Query, description = "Continue after this storage ID of the previous page"), + ("limit" = Option, Query, description = "Events to scan; defaults to 200, clamped to 1..500")), + responses((status = 200, body = PersonPlatformEventPage, description = "One scan of stored platform events newest first, without heartbeats: server-wide ones, and those of targets the person lists"), + (status = 401, description = "Session rejected")), security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn platform_events( + (State(store), Query(options)): (State, Query), + Extension(default_role): Extension, + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result, ApiError> { + Ok(Json( + list_person_platform_events( + &store, + principal, + (options.before, options.limit.unwrap_or(200)), + default_role, + ) + .await?, + )) +} + fn not_found(message: &str) -> ApiError { ApiError { status: StatusCode::NOT_FOUND, diff --git a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs index 1397e69ed2..87447e8f93 100644 --- a/crates/sqlbuild-orchestration/src/api/main/serve_api.rs +++ b/crates/sqlbuild-orchestration/src/api/main/serve_api.rs @@ -8,23 +8,18 @@ use crate::api::_helpers::resources::server_errors::generic_server_errors; use crate::api::_helpers::resources::ui_assets::{UI_ASSETS, serve_ui}; use crate::api::_helpers::resources::{changes, router::router}; use crate::api::main::openapi_document::openapi_document; -use crate::api::models::{ApiAuthentication, ChangeFeed, DevelopmentScope, GroupRole}; +use crate::api::models::{ApiAuthentication, ChangeFeed, DevelopmentScope, GroupRole, ServerFacts}; use crate::authorization::main::publish_group_roles::publish_group_roles; use crate::runs::types::RunHandoff; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; use crate::store::main::subscribe_changes::subscribe_changes; use crate::store::models::ControlStore; -use crate::warehouse_metadata::models::StatisticsRetention; pub async fn serve_api( store: ControlStore, listener: tokio::net::TcpListener, - (authentication, (images, planner), retention): ( - ApiAuthentication, - RunHandoff, - StatisticsRetention, - ), + (authentication, (images, planner), facts): (ApiAuthentication, RunHandoff, ServerFacts), (ready, shutdown): ( impl FnOnce() + Send + 'static, impl Future + Send + 'static, @@ -44,7 +39,8 @@ pub async fn serve_api( .merge(authentication_routes.layer(Extension(feed))) .layer(Extension(std::sync::Arc::new(images))) .layer(Extension(planner)) - .layer(Extension(retention)) + .layer(Extension(facts.statistics)) + .layer(Extension(facts)) .route( "/api/openapi.json", get(|| async { Json(openapi_document()) }), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index cb27eb28ea..dd7cd5a91b 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -7,7 +7,7 @@ use crate::api::_helpers::authentication::{cli_login, oidc}; use crate::api::_helpers::resources::{ administrators, audit, changes, location_health, location_work, person_bootstrap, person_checks, person_location_work, person_maintenance, person_node_sql, person_operations, - person_reads, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, + person_plans, person_reads, run_reads, runs, snapshots, trigger_reads, warehouse_metadata, }; #[derive(Debug, utoipa::OpenApi)] @@ -17,6 +17,7 @@ use crate::api::_helpers::resources::{ administrators::people, administrators::holdings, administrators::definitions, + administrators::server, audit::read, person_operations::request_job, person_operations::request_adhoc, @@ -38,9 +39,11 @@ use crate::api::_helpers::resources::{ person_reads::triggers, person_reads::ticks, person_reads::trigger_changes, + person_reads::sensor_state, person_reads::snapshots, person_reads::health, person_reads::problems, + person_reads::platform_events, person_operations::catalog, person_operations::statistics, person_operations::fingerprint, @@ -55,6 +58,8 @@ use crate::api::_helpers::resources::{ person_maintenance::coverage, person_location_work::request, person_location_work::result, + person_plans::request, + person_plans::read, person_node_sql::read, password_sessions::login, password_sessions::current, @@ -93,6 +98,25 @@ use crate::api::_helpers::resources::{ ))] pub(crate) struct ControlApi; +/// A page of runs, newest first; the `before_*` pair names the last run of the previous page. `queued` lists the queue instead. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RunsQuery { + pub(crate) limit: Option, + #[serde(default)] + pub(crate) queued: bool, + pub(crate) before_queued_at: Option>, + pub(crate) before_run_id: Option, +} + +/// A page of platform events, newest first; `before` is the last storage ID of the previous page. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct PlatformEventsQuery { + pub(crate) before: Option, + pub(crate) limit: Option, +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct ReadQuery { @@ -477,3 +501,38 @@ pub(crate) struct BoundedReader { pub(crate) remaining: usize, pub(crate) exceeded: bool, } + +/// What the server was started with, reported by the settings read. +#[derive(Clone, Debug, Default)] +pub struct ServerFacts { + pub version: String, + /// Whether this process runs the scheduler loop. + pub scheduler: bool, + pub audit: crate::credentials::models::AuditRetention, + pub statistics: crate::warehouse_metadata::models::StatisticsRetention, + pub snapshots: crate::snapshots::models::SnapshotRetention, +} + +/// The server's version, storage, sign-in and retention, for administrators. +#[derive(Clone, Debug, Serialize, utoipa::ToSchema)] +pub struct ServerSettings { + pub version: String, + pub login_mode: LoginMode, + /// Whether this server runs the scheduler; another server may hold the lease. + pub scheduler: bool, + pub installation: crate::platform::models::InstallationRecord, + pub retention: ServerRetention, +} + +/// How long the server keeps each kind of record. +#[derive(Clone, Debug, Serialize, utoipa::ToSchema)] +pub struct ServerRetention { + pub audit_days: i64, + pub statistics_days: i64, + /// Publications kept per location target. + pub snapshots_kept: i64, + pub platform_alert_days: i64, + pub heartbeat_days: i64, + pub location_health_days: i64, + pub sensor_evaluation_days: i64, +} diff --git a/crates/sqlbuild-orchestration/src/authorization/main/mod.rs b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs index 6237fd40a6..761e1fd373 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/mod.rs @@ -11,7 +11,7 @@ pub mod grant_admin; pub mod list_role_definitions; pub mod list_role_holdings; pub(crate) mod person_assigns_roles; -pub(crate) mod person_lists_target; +pub(crate) mod person_holds_target; pub(crate) mod publish_custom_roles; pub(crate) mod publish_group_roles; pub(crate) mod read_person_access; diff --git a/crates/sqlbuild-orchestration/src/authorization/main/person_lists_target.rs b/crates/sqlbuild-orchestration/src/authorization/main/person_holds_target.rs similarity index 82% rename from crates/sqlbuild-orchestration/src/authorization/main/person_lists_target.rs rename to crates/sqlbuild-orchestration/src/authorization/main/person_holds_target.rs index c4d8d5644f..ef5f69b46a 100644 --- a/crates/sqlbuild-orchestration/src/authorization/main/person_lists_target.rs +++ b/crates/sqlbuild-orchestration/src/authorization/main/person_holds_target.rs @@ -1,4 +1,4 @@ -//! Whether a person may still receive a target's change notices; unaudited, rechecked before every notice. +//! Whether a person holds an action on every node of a target, unaudited: change notices recheck view before every notice, and reads trim detail with it. use uuid::Uuid; @@ -11,11 +11,11 @@ use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit, query::query}; use crate::store::models::{ControlStore, Parameter, TransactionMode}; -/// Whether an active person still views every node of the target, as its listings require; unaudited, for change notices. -pub(crate) async fn person_lists_target( +/// Whether an active person holds `action` on every current node of the target; with view, as listings require. +pub(crate) async fn person_holds_target( store: &ControlStore, principal_id: Uuid, - (location_id, target): (Uuid, &str), + (action, location_id, target): (Action, Uuid, &str), default_role: DefaultRole, ) -> Result { let transaction = begin(store, TransactionMode::Read).await?; @@ -43,7 +43,7 @@ pub(crate) async fn person_lists_target( return Ok(false); }; let scope = PermissionScope { - action: Action::View, + action, location_id: Some(location_id), target: Some(target.into()), nodes, diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index f2027c65c2..d965dae987 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -126,6 +126,10 @@ impl AuditRetention { pub(crate) fn cutoff(self, now: DateTime) -> DateTime { now - chrono::Duration::days(i64::from(self.days)) } + + pub(crate) fn kept_days(self) -> u16 { + self.days + } } impl Default for AuditRetention { diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/job_detail.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/job_detail.rs index 7eaeb7eda8..fb4cc6ad7b 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/job_detail.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/job_detail.rs @@ -91,3 +91,16 @@ pub(crate) fn critical_nodes(nodes: &BTreeMap) -> BTreeSet .map(|(key, _)| key.clone()) .collect() } + +/// The one cursor type of the selected nodes that have a cursor, so a backfill window can be typed. +pub(crate) fn cursor_type(nodes: &BTreeMap) -> Option { + let types: BTreeSet<&str> = nodes + .values() + .filter_map(|node| node.definition["config"]["cursor_type"].as_str()) + .collect(); + let mut types = types.into_iter(); + match (types.next(), types.next()) { + (Some(only), None) => Some(only.to_owned()), + _ => None, + } +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs index c527a4066d..ffe4978047 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs @@ -86,6 +86,8 @@ pub(crate) async fn jobs_in( selection: job.definition_json.selection, concurrency_groups: job.definition_json.concurrency_groups, snapshot_id: job.snapshot_id, + select: job.definition_json.select, + declared_at: job.definition_json.declared_at, }) .collect(); Ok((transaction, jobs)) diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs index f75bdfa563..aa906942d8 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs @@ -46,8 +46,10 @@ pub(crate) async fn trigger_scope( priority: 0, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }; - let (transaction, frozen) = frozen_job_request(transaction, context, &job).await?; + let (transaction, frozen) = frozen_job_request(transaction, (context, None), &job).await?; let (transaction, nodes) = job_write_set(transaction, frozen.snapshot_id, &frozen.selection).await?; Ok(( diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs index b258882455..d000185371 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_state.rs @@ -2,8 +2,9 @@ use chrono::{DateTime, TimeDelta, Utc}; +use crate::declarations::constants::UPCOMING_TIMES; use crate::declarations::models::{ - PreviousTriggerState, Schedule, TriggerDefinition, TriggerStatus, + PreviousTriggerState, Schedule, TriggerDeclaration, TriggerDefinition, TriggerStatus, }; use crate::scheduler::constants::MAX_TICKS_PER_BATCH; use crate::scheduler::main::next_schedule_time::next_schedule_time; @@ -62,3 +63,21 @@ pub(crate) fn trigger_timing( }; Ok((status, next)) } + +/// The next due times of a running schedule from `next_due_at` on, as the scheduler works them out. +pub(crate) fn upcoming_times( + declaration: &TriggerDeclaration, + next_due_at: Option>, +) -> Result>, ControlError> { + let (TriggerDeclaration::Schedule(definition), Some(first)) = (declaration, next_due_at) else { + return Ok(Vec::new()); + }; + let mut times: Vec> = vec![first]; + while times.len() < UPCOMING_TIMES { + let Some(next) = next_schedule_time(&definition.schedule, times[times.len() - 1])? else { + break; + }; + times.push(next); + } + Ok(times) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/constants.rs b/crates/sqlbuild-orchestration/src/declarations/constants.rs index 00fc324482..125a59e986 100644 --- a/crates/sqlbuild-orchestration/src/declarations/constants.rs +++ b/crates/sqlbuild-orchestration/src/declarations/constants.rs @@ -6,3 +6,6 @@ pub(crate) const VARS_SETTING: &str = "vars"; pub(crate) const WITHHELD_SETTING: &str = "[REDACTED]"; /// Settings that name another target; a value that isn't a target name, such as a state path, is withheld. pub(crate) const DEFER_SETTING_PREFIX: &str = "defer_"; + +/// How many due times a person's trigger listing gives for each running schedule. +pub(crate) const UPCOMING_TIMES: usize = 5; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs b/crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs index 3e178777ac..94a4f0c976 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs @@ -7,6 +7,7 @@ use uuid::Uuid; use crate::authorization::main::authorize_listing::authorize_listing; use crate::credentials::models::{AuditContext, DefaultRole}; use crate::declarations::_helpers::reads::{latest_ticks_in, triggers_in}; +use crate::declarations::_helpers::trigger_state::upcoming_times; use crate::declarations::models::PersonTriggerRecord; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; @@ -32,11 +33,14 @@ pub async fn list_person_triggers( .into_iter() .map(|tick| (tick.trigger_id, tick)) .collect(); - Ok(triggers + triggers .into_iter() - .map(|trigger| PersonTriggerRecord { - latest_tick: latest.remove(&trigger.trigger_id), - trigger, + .map(|trigger| { + Ok(PersonTriggerRecord { + latest_tick: latest.remove(&trigger.trigger_id), + upcoming: upcoming_times(&trigger.definition_json, trigger.next_due_at)?, + trigger, + }) }) - .collect()) + .collect() } diff --git a/crates/sqlbuild-orchestration/src/declarations/main/read_person_job.rs b/crates/sqlbuild-orchestration/src/declarations/main/read_person_job.rs index abc034ffe5..6ffb2a9d36 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/read_person_job.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/read_person_job.rs @@ -5,7 +5,7 @@ use uuid::Uuid; use crate::authorization::main::authorize_listing::authorize_listing; use crate::credentials::models::{AuditContext, DefaultRole}; use crate::declarations::_helpers::job_detail::{ - critical_nodes, groups_running, members_in_order, shown_settings, + critical_nodes, cursor_type, groups_running, members_in_order, shown_settings, }; use crate::declarations::main::frozen_definitions::frozen_definitions; use crate::declarations::models::{JobDetailRecord, StoredJob}; @@ -60,8 +60,13 @@ pub async fn read_person_job( members, member_of: groups_running(&definitions, job_key), critical_nodes: critical_nodes(&nodes), + cursor_type: cursor_type(&nodes), selection: job.selection, concurrency_groups: job.concurrency_groups, + select: job.select, + exclude: job.exclude, + targets: job.targets, + declared_at: job.declared_at, })) } diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 66e289e3cd..b67dadf618 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -35,6 +35,24 @@ pub struct JobDefinition { pub settings: BTreeMap, #[serde(default)] pub concurrency_groups: BTreeSet, + /// The selectors as written, for display; `selection` is what runs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub select: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exclude: Option, + /// The targets the declaration is limited to; empty for every target. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub targets: BTreeSet, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub declared_at: Option, +} + +/// Where a declaration was written, relative to the project root. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct DeclaredAt { + pub path: String, + pub line: u32, } /// A virtual parent shares run controls, but only its physical children claim node writers. @@ -68,6 +86,11 @@ pub struct TriggerDefinition { pub starts_at: Option>, #[serde(default)] pub default_status: TriggerStatus, + /// The targets the declaration is limited to; empty for every target. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub targets: BTreeSet, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub declared_at: Option, } #[derive(Clone, Debug, Default, PartialEq)] @@ -97,6 +120,9 @@ pub struct JobRecord { pub selection: BTreeSet, pub concurrency_groups: BTreeSet, pub snapshot_id: uuid::Uuid, + /// A job's selectors as written; absent for a group and for snapshots published before they were recorded. + pub select: Option, + pub declared_at: Option, } /// A `sqb_jobs` row as stored. @@ -125,6 +151,13 @@ pub struct JobDetailRecord { pub member_of: BTreeSet, /// Selected nodes whose failure stops a group. pub critical_nodes: BTreeSet, + pub select: Option, + pub exclude: Option, + /// Empty when the declaration applies to every target. + pub targets: BTreeSet, + pub declared_at: Option, + /// The cursor type every selected cursor node shares; absent when none has a cursor or types differ. + pub cursor_type: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] @@ -151,6 +184,8 @@ pub struct PersonTriggerRecord { #[serde(flatten)] pub trigger: TriggerRecord, pub latest_tick: Option, + /// A running schedule's next due times from `next_due_at` on, at most five; empty otherwise. + pub upcoming: Vec>, } /// One operator change of a trigger, as a person reads it: what changed, who changed it, when and why. diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/event_history.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/event_history.rs new file mode 100644 index 0000000000..b62df08dbf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/event_history.rs @@ -0,0 +1,108 @@ +//! Stored platform events newest first, each with the location target it concerns. + +use std::collections::BTreeMap; + +use uuid::Uuid; + +use crate::platform::models::{ + EventScope, PersonPlatformEvent, PlatformEvent, PlatformEventEnvelope, +}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query, query_records::query_records}; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Up to `limit` events before `before` without heartbeats or orphans, targets resolved, and where a further scan continues. +pub(crate) async fn event_history( + transaction: ControlTransaction, + (before, limit): (Option, u32), +) -> Result<(ControlTransaction, Vec, Option), ControlError> { + let limit: u32 = limit.clamp(1, 500); + let mut parameters = vec![Parameter::Integer(i64::from(limit))]; + let older = before.map_or("", |storage_id| { + parameters.push(Parameter::Integer(storage_id)); + "AND storage_id<$2 " + }); + let (transaction, events): (ControlTransaction, Vec) = query_records(transaction, &format!( + "SELECT storage_id,event_id,occurred_at,payload FROM recall_events WHERE producer='platform' \ + AND invocation_id LIKE 'platform/%' AND event_type<>'platform_heartbeat' {older}ORDER BY storage_id DESC LIMIT $1"), + parameters).await?; + let mut triggers: Vec = Vec::new(); + let mut runs: Vec = Vec::new(); + for envelope in &events { + match envelope.payload.platform { + PlatformEvent::TickOverdue { trigger_id, .. } + | PlatformEvent::CalendarHorizonLow { trigger_id, .. } => triggers.push(trigger_id), + PlatformEvent::LaunchFailed { run_id, .. } => runs.push(run_id), + _ => {} + } + } + let (transaction, mut scopes) = + scopes_of(transaction, ("sqb_triggers", "trigger_id"), &triggers).await?; + let (transaction, run_scopes) = scopes_of(transaction, ("sqb_runs", "run_id"), &runs).await?; + scopes.extend(run_scopes); + let next: Option = (events.len() == limit as usize) + .then(|| events.last().map(|envelope| envelope.storage_id)) + .flatten(); + let history: Vec = events + .into_iter() + .filter_map(|envelope| person_event(envelope, &scopes)) + .collect(); + Ok((transaction, history, next)) +} + +fn person_event( + envelope: PlatformEventEnvelope, + scopes: &BTreeMap, +) -> Option { + let scope: Option<(Uuid, String)> = match &envelope.payload.platform { + PlatformEvent::LocationBroken { + location_id, + target, + .. + } => Some((*location_id, target.clone())), + PlatformEvent::TickOverdue { trigger_id, .. } + | PlatformEvent::CalendarHorizonLow { trigger_id, .. } => { + Some(scopes.get(trigger_id).cloned()?) + } + PlatformEvent::LaunchFailed { run_id, .. } => Some(scopes.get(run_id).cloned()?), + PlatformEvent::Heartbeat { .. } | PlatformEvent::SchedulerNoLeader => None, + }; + let (location_id, target) = scope.map_or((None, None), |(location_id, target)| { + (Some(location_id), Some(target)) + }); + Some(PersonPlatformEvent { + storage_id: envelope.storage_id, + event_id: envelope.event_id, + occurred_at: envelope.occurred_at, + location_id, + target, + event: envelope.payload.platform, + }) +} + +async fn scopes_of( + transaction: ControlTransaction, + (table, column): (&str, &str), + ids: &[Uuid], +) -> Result<(ControlTransaction, BTreeMap), ControlError> { + if ids.is_empty() { + return Ok((transaction, BTreeMap::new())); + } + let (transaction, rows) = query( + transaction, + &format!( + "SELECT {column} AS id, location_id, target FROM {table} WHERE {column} IN ({})", + Parameter::placeholders(1..=ids.len()) + ), + ids.iter().copied().map(Parameter::Id).collect(), + ) + .await?; + let scopes: BTreeMap = rows + .into_iter() + .map(decode_row) + .collect::, _>>()? + .into_iter() + .map(|scope| (scope.id, (scope.location_id, scope.target))) + .collect(); + Ok((transaction, scopes)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs index 51c5fd6959..d0298eee0d 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/liveness.rs @@ -4,6 +4,7 @@ use chrono::TimeDelta; use uuid::Uuid; use crate::platform::_helpers::observations::observe; +use crate::platform::constants::LOCATION_SILENCE_SECONDS; use crate::platform::models::{PlatformEvent, WatchdogLocation}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -19,7 +20,9 @@ pub(crate) async fn scan_location_page( after: Option<(Uuid, String)>, ) -> Result, ControlError> { let transaction = begin(store, TransactionMode::Write).await?; - let mut parameters = vec![Parameter::Time(transaction.now - TimeDelta::seconds(90))]; + let mut parameters = vec![Parameter::Time( + transaction.now - TimeDelta::seconds(LOCATION_SILENCE_SECONDS), + )]; let cursor = if let Some((location_id, target)) = after { parameters.extend([Parameter::Id(location_id), Parameter::Text(target)]); "AND (i.location_id>$2 OR (i.location_id=$2 AND i.target>$3))" diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs index a302fe0aee..f2192f0d44 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod connection_tests; +pub(crate) mod event_history; pub(crate) mod liveness; pub(crate) mod location_heartbeat; pub(crate) mod observations; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/problems.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/problems.rs index f8e57a9574..ded3ac32a8 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/problems.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/problems.rs @@ -3,7 +3,9 @@ use chrono::{DateTime, TimeDelta, Utc}; use uuid::Uuid; -use crate::platform::models::{LateTarget, OverdueTick, ProblemScan}; +use crate::declarations::models::Schedule; +use crate::platform::constants::LOCATION_SILENCE_SECONDS; +use crate::platform::models::{LateTarget, LowCalendar, OverdueTick, ProblemScan, RunningSchedule}; use crate::store::errors::ControlError; use crate::store::main::decode_row::decode_row; use crate::store::main::query::query; @@ -25,7 +27,7 @@ pub(crate) async fn scan_problems(store: &ControlStore) -> Result Result = schedules + .into_iter() + .map(decode_row) + .collect::>()?; + let low_calendars: Vec = schedules + .into_iter() + .filter_map(|schedule| { + let (refresh_at, horizon_end) = calendar_low(&schedule.definition_json.schedule, now)?; + Some(LowCalendar { + trigger_id: schedule.trigger_id, + name: schedule.name, + location_id: schedule.location_id, + location_name: schedule.location_name, + target: schedule.target, + refresh_at, + horizon_end, + }) + }) + .collect(); Ok(ProblemScan { + low_calendars, now, lease: leases.into_iter().next().map(decode_row).transpose()?, broken: locations @@ -95,3 +127,16 @@ pub(crate) async fn late_ticks( fn overdue_before(now: DateTime) -> DateTime { now - TimeDelta::seconds(30) } + +/// A calendar is low when its last date is at most a day away; gives when it refreshes and its last date. +pub(crate) fn calendar_low( + schedule: &Schedule, + now: DateTime, +) -> Option<(DateTime, Option>)> { + let Schedule::Calendar { times, refresh_at } = schedule else { + return None; + }; + let last: Option> = times.last().copied(); + last.is_none_or(|last| last <= now + TimeDelta::days(1)) + .then_some((*refresh_at, last)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs index 2bfb02d6df..a85f056c46 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/watchdog.rs @@ -4,9 +4,9 @@ use chrono::TimeDelta; use tokio::sync::watch; use uuid::Uuid; -use crate::declarations::models::Schedule; use crate::platform::_helpers::liveness::scan_location_page; use crate::platform::_helpers::observations::observe; +use crate::platform::_helpers::problems::calendar_low; use crate::platform::models::{InstallationHealth, PlatformEvent, WatchdogTrigger}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; @@ -113,10 +113,8 @@ async fn scan_trigger_page( ) .await?; } - if let Schedule::Calendar { times, refresh_at } = trigger.definition_json.schedule - && times - .last() - .is_none_or(|last| *last <= now + TimeDelta::days(1)) + if let Some((refresh_at, horizon_end)) = + calendar_low(&trigger.definition_json.schedule, now) { transaction = observe( transaction, @@ -125,7 +123,7 @@ async fn scan_trigger_page( PlatformEvent::CalendarHorizonLow { trigger_id: trigger.trigger_id, refresh_at, - horizon_end: times.last().copied(), + horizon_end, }, None, ), diff --git a/crates/sqlbuild-orchestration/src/platform/constants.rs b/crates/sqlbuild-orchestration/src/platform/constants.rs index acf2969d6f..2753bdb529 100644 --- a/crates/sqlbuild-orchestration/src/platform/constants.rs +++ b/crates/sqlbuild-orchestration/src/platform/constants.rs @@ -19,3 +19,18 @@ pub(crate) const SENSOR_EVALUATION_RETENTION_DAYS: i64 = 7; /// Bounded retention deletions per transaction. pub(crate) const HEARTBEAT_PAGE: usize = 1000; + +/// A plan document larger than this is refused by the server and replaced by an error on the location. +pub const PLAN_DOCUMENT_BYTES: usize = 8 * 1024 * 1024; + +/// How long a location may hold a plan before another instance may claim it. +pub(crate) const PLAN_LEASE_SECONDS: i64 = 300; + +/// A plan nobody claims or finishes by then is reported as expired. +pub(crate) const PLAN_EXPIRY_SECONDS: i64 = 600; + +/// Unfinished plans one target accepts across everyone; each person may have one. +pub(crate) const PLAN_QUEUE_LIMIT: i64 = 4; + +/// A location instance unseen for longer than this cannot take a plan; locations report every 30 seconds. +pub(crate) const LOCATION_SILENCE_SECONDS: i64 = 90; diff --git a/crates/sqlbuild-orchestration/src/platform/main/_read_target_work.rs b/crates/sqlbuild-orchestration/src/platform/main/_read_target_work.rs new file mode 100644 index 0000000000..717a6bd3dd --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/_read_target_work.rs @@ -0,0 +1,19 @@ +//! Read one request as a person reads it: only when it belongs to the location and target they named. + +use uuid::Uuid; + +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::LocationWorkRecord; +use crate::store::errors::ControlError; +use crate::store::models::ControlTransaction; + +pub(in crate::platform) async fn read_target_work( + transaction: ControlTransaction, + (location_id, target, request_id): (Uuid, &str, Uuid), +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, stored) = read_location_work(transaction, request_id).await?; + let record = stored.map(|stored| stored.record).filter(|record| { + record.request_json.location_id == location_id && record.request_json.target == target + }); + Ok((transaction, record)) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs index 106f446a4c..0337aecc42 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/claim_location_work.rs @@ -1,11 +1,12 @@ -//! Serialized leases fence results from replaced location instances and uncertain poll retries. +//! Serialized leases fence results from replaced location instances and uncertain poll retries; plans go last. use chrono::Duration; use uuid::Uuid; +use crate::platform::constants::PLAN_LEASE_SECONDS; use crate::platform::main::read_location_work::read_location_work; use crate::platform::main::require_instance::require_instance; -use crate::platform::models::{LocationWorkClaim, LocationWorkPoll}; +use crate::platform::models::{LocationWork, LocationWorkClaim, LocationWorkPoll}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -27,7 +28,7 @@ pub async fn claim_location_work( "SELECT w.request_id FROM sqb_location_work w WHERE w.location_id=$1 AND w.target=$2 \ AND w.expires_at>$3 AND w.completed_at IS NULL \ AND (w.claimed_until IS NULL OR w.claimed_until<=$3 OR w.session_id=$4) \ - AND (w.request_json->'work'->>'kind'='check_connection' OR EXISTS ( \ + AND (w.request_json->'work'->>'kind' IN ('check_connection','plan') OR EXISTS ( \ SELECT 1 FROM sqb_sensor_evaluation_events e JOIN sqb_triggers t ON t.trigger_id=e.trigger_id \ JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id JOIN sqb_scheduler_lease l ON l.name='scheduler' \ WHERE e.evaluation_id=w.request_id AND e.phase='requested' AND s.status='running' \ @@ -35,7 +36,7 @@ pub async fn claim_location_work( AND l.epoch=CAST(e.fact_json->'request'->>'epoch' AS BIGINT) \ AND NOT EXISTS (SELECT 1 FROM sqb_sensor_evaluation_events f \ WHERE f.evaluation_id=e.evaluation_id AND f.phase<>'requested'))) \ - ORDER BY w.requested_at, w.request_id LIMIT 1", + ORDER BY CASE WHEN w.request_json->'work'->>'kind'='plan' THEN 1 ELSE 0 END, w.requested_at, w.request_id LIMIT 1", vec![Parameter::Id(location_id), Parameter::Text(poll.target.clone()), Parameter::Time(now), Parameter::Id(poll.session_id)]).await?; let Some(row) = rows.first() else { commit(transaction).await?; @@ -45,6 +46,11 @@ pub async fn claim_location_work( .map_err(|error| ControlError::Invalid(format!("invalid work identity: {error}")))?; let (transaction, stored) = read_location_work(transaction, request_id).await?; let stored = stored.ok_or_else(|| ControlError::Invalid("location work disappeared".into()))?; + let lease = if matches!(stored.record.request_json.work, LocationWork::Plan { .. }) { + PLAN_LEASE_SECONDS + } else { + 60 + }; let retained = stored.session_id == Some(poll.session_id) && stored.claimed_until.is_some_and(|until| until > now); let (claim_id, lease_expires_at) = if retained { @@ -55,7 +61,7 @@ pub async fn claim_location_work( } else { ( Uuid::now_v7(), - (now + Duration::seconds(60)).min(stored.record.expires_at), + (now + Duration::seconds(lease)).min(stored.record.expires_at), ) }; let (transaction, _) = query(transaction, diff --git a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs index 8af79d1995..cc19fc50ad 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/complete_location_work.rs @@ -1,8 +1,10 @@ -//! A completion is immutable; only the current unexpired claim may publish its result. +//! A completion is immutable; only the current unexpired claim may publish its result, redacted. -use serde_json::json; +use serde_json::{Value, json}; +use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; use uuid::Uuid; +use crate::platform::constants::PLAN_DOCUMENT_BYTES; use crate::platform::main::read_location_work::read_location_work; use crate::platform::main::require_instance::require_instance; use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkResult}; @@ -19,6 +21,7 @@ pub async fn complete_location_work( (location_id, request_id): (Uuid, Uuid), result: &LocationWorkResult, ) -> Result { + let result = &redacted(result); let transaction = begin(store, TransactionMode::Write).await?; let (transaction, stored) = read_location_work(transaction, request_id).await?; let stored = stored @@ -51,9 +54,53 @@ pub async fn complete_location_work( { return Err(ControlError::Fenced); } - let valid = match (&stored.record.request_json.work, &result.outcome) { + let valid = valid_outcome(&stored.record.request_json.work, &result.outcome); + if !valid { + return Err(ControlError::Invalid( + "location result does not match its work".into(), + )); + } + let now = transaction.now; + let (transaction, _) = query(transaction, + "UPDATE sqb_location_work SET result_json=$2, completed_at=$3 WHERE request_id=$1 AND completed_at IS NULL", + vec![Parameter::Id(request_id), Parameter::Json(json!(result)), Parameter::Time(now)]).await?; + commit(transaction).await?; + Ok(true) +} + +/// Warehouse text in a plan's error and warnings is redacted before it is compared or stored. +fn redacted(result: &LocationWorkResult) -> LocationWorkResult { + let mut result = result.clone(); + if let LocationWorkOutcome::Plan { document } = &mut result.outcome { + let texts = ["/error/message", "/error/help"] + .into_iter() + .map(str::to_owned) + .chain( + (0..document["warnings"].as_array().map_or(0, Vec::len)) + .map(|index| format!("/warnings/{index}/message")), + ) + .collect::>(); + for pointer in texts { + if let Some(Value::String(text)) = document.pointer_mut(&pointer) { + *text = redact_error_text(text); + } + } + } + result +} + +/// Whether the outcome answers the work it claims to, within the bounds the server accepts. +fn valid_outcome(work: &LocationWork, outcome: &LocationWorkOutcome) -> bool { + match (work, outcome) { (_, LocationWorkOutcome::Superseded {}) | (LocationWork::CheckConnection {}, LocationWorkOutcome::Connection { .. }) => true, + (LocationWork::Plan { .. }, LocationWorkOutcome::Plan { document }) => { + document + .as_object() + .is_some_and(|plan| plan.contains_key("models") || plan.contains_key("error")) + && serde_json::to_vec(document) + .is_ok_and(|bytes| bytes.len() <= PLAN_DOCUMENT_BYTES) + } ( LocationWork::EvaluateSensor { .. }, LocationWorkOutcome::SensorEvaluation { outcome }, @@ -66,16 +113,5 @@ pub async fn complete_location_work( CustomSensorOutcome::Failed | CustomSensorOutcome::TimedOut => true, }, _ => false, - }; - if !valid { - return Err(ControlError::Invalid( - "location result does not match its work".into(), - )); } - let now = transaction.now; - let (transaction, _) = query(transaction, - "UPDATE sqb_location_work SET result_json=$2, completed_at=$3 WHERE request_id=$1 AND completed_at IS NULL", - vec![Parameter::Id(request_id), Parameter::Json(json!(result)), Parameter::Time(now)]).await?; - commit(transaction).await?; - Ok(true) } diff --git a/crates/sqlbuild-orchestration/src/platform/main/enqueue_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/enqueue_location_work.rs new file mode 100644 index 0000000000..39444d4b20 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/enqueue_location_work.rs @@ -0,0 +1,56 @@ +//! Pin new location work to the current publication and audit the request in the caller's transaction. + +use chrono::Duration; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::audit::append_audit::append_audit; +use crate::credentials::models::AuditContext; +use crate::platform::models::LocationWorkRequest; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// The caller has already authorised the request and ruled out a retry of the same request ID. +pub(crate) async fn enqueue_location_work( + transaction: ControlTransaction, + context: &AuditContext, + request: &LocationWorkRequest, + lifetime: Duration, +) -> Result { + let (transaction, rows) = query(transaction, + "SELECT s.snapshot_id, s.generation FROM sqb_current_snapshots c \ + JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id JOIN sqb_locations l ON l.location_id=c.location_id \ + WHERE c.location_id=$1 AND c.target=$2 AND l.archived_at IS NULL", + vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone())]).await?; + let snapshot = rows + .first() + .ok_or_else(|| ControlError::Invalid("location work requires a published target".into()))?; + let snapshot_id = serde_json::from_value(snapshot["snapshot_id"].clone()) + .map_err(|error| ControlError::Invalid(format!("invalid snapshot identity: {error}")))?; + let generation = snapshot["generation"] + .as_i64() + .ok_or_else(|| ControlError::Invalid("invalid snapshot generation".into()))?; + let now = transaction.now; + let (transaction, _) = query(transaction, + "INSERT INTO sqb_location_work \ + (request_id,location_id,target,snapshot_id,generation,request_json,requested_by,requested_at,expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", + vec![Parameter::Id(request.request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), + Parameter::Id(snapshot_id), Parameter::Integer(generation), Parameter::Json(json!(request)), + Parameter::Text(context.actor.clone()), Parameter::Time(now), Parameter::Time(now + lifetime)]).await?; + append_audit( + transaction, + context, + ( + Uuid::new_v5(&request.request_id, b"location-work-requested"), + "location.work_requested", + ), + ( + json!({"location_id": request.location_id, "target": request.target}), + json!(null), + json!(request), + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/list_person_platform_events.rs b/crates/sqlbuild-orchestration/src/platform/main/list_person_platform_events.rs new file mode 100644 index 0000000000..b9691f11f8 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/list_person_platform_events.rs @@ -0,0 +1,54 @@ +//! People see server-wide platform events, and a location target's events only for targets they list. + +use std::collections::BTreeMap; + +use uuid::Uuid; + +use crate::authorization::main::person_holds_target::person_holds_target; +use crate::credentials::models::{Action, DefaultRole}; +use crate::platform::_helpers::event_history::event_history; +use crate::platform::models::{PersonPlatformEvent, PersonPlatformEventPage}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +/// Newest first: one scan of `limit` stored events, keeping those the person may see, so a page can be empty yet continue. +pub async fn list_person_platform_events( + store: &ControlStore, + principal_id: Uuid, + page: (Option, u32), + default_role: DefaultRole, +) -> Result { + let (transaction, history, before) = + event_history(begin(store, TransactionMode::Read).await?, page).await?; + commit(transaction).await?; + let mut listed: BTreeMap<(Uuid, String), bool> = BTreeMap::new(); + let mut visible: Vec = Vec::new(); + for event in history { + let (Some(location_id), Some(target)) = (event.location_id, event.target.clone()) else { + visible.push(event); + continue; + }; + let shown = match listed.get(&(location_id, target.clone())) { + Some(shown) => *shown, + None => { + let shown = person_holds_target( + store, + principal_id, + (Action::View, location_id, &target), + default_role, + ) + .await?; + listed.insert((location_id, target), shown); + shown + } + }; + if shown { + visible.push(event); + } + } + Ok(PersonPlatformEventPage { + events: visible, + before, + }) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/mod.rs b/crates/sqlbuild-orchestration/src/platform/main/mod.rs index d1c062ba19..f075ce27f5 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/mod.rs @@ -1,11 +1,16 @@ +pub(in crate::platform) mod _read_target_work; pub mod claim_location_work; pub mod complete_location_work; pub mod connect_location_work; +pub(crate) mod enqueue_location_work; +pub mod list_person_platform_events; pub mod location_health; pub mod location_work; +pub mod read_installation; pub(crate) mod read_location_work; pub mod read_person_connection_test; pub mod read_person_location_health; +pub mod read_person_plan; pub mod read_person_problems; pub(crate) mod read_platform_events; pub mod report_location_health; diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs b/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs new file mode 100644 index 0000000000..6e20822bba --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_installation.rs @@ -0,0 +1,45 @@ +//! Administrators read when the control store was created and which schema it holds. + +use uuid::Uuid; + +use crate::authorization::main::begin_authorized::begin_authorized; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::platform::models::{InstallationRecord, StorageKind, StoredInstallation}; +use crate::store::errors::ControlError; +use crate::store::main::{commit::commit, query_records::query_records}; +use crate::store::models::{ControlStore, ControlTransaction}; + +/// Requires `user.manage` everywhere, as the people list does. +pub async fn read_installation( + store: &ControlStore, + credentials: (&AuditContext, Uuid), +) -> Result { + let (transaction, _) = begin_authorized( + store, + credentials, + &PermissionScope::action(Action::UserManage, None), + DefaultRole::None, + ) + .await?; + let (transaction, rows): (ControlTransaction, Vec) = query_records( + transaction, + "SELECT i.created_at AS installed_at, v.checksum AS schema_checksum, v.applied_at AS schema_applied_at \ + FROM sqb_installation i JOIN sqb_schema_version v ON v.singleton=i.singleton WHERE i.singleton=1", + Vec::new(), + ) + .await?; + commit(transaction).await?; + let stored = rows.into_iter().next().ok_or_else(|| { + ControlError::Invalid("the control store has no installation record".into()) + })?; + let storage: StorageKind = match store { + ControlStore::Sqlite(_) => StorageKind::Sqlite, + ControlStore::Postgres(_) => StorageKind::Postgresql, + }; + Ok(InstallationRecord { + storage, + installed_at: stored.installed_at, + schema_checksum: stored.schema_checksum, + schema_applied_at: stored.schema_applied_at, + }) +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_person_connection_test.rs b/crates/sqlbuild-orchestration/src/platform/main/read_person_connection_test.rs index 1bd5f7adbc..7df117547d 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/read_person_connection_test.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/read_person_connection_test.rs @@ -4,7 +4,7 @@ use uuid::Uuid; use crate::credentials::models::AuditContext; use crate::platform::_helpers::connection_tests::authorize_connection_test; -use crate::platform::main::read_location_work::read_location_work; +use crate::platform::main::_read_target_work::read_target_work; use crate::platform::models::{LocationWork, LocationWorkRecord}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; @@ -21,12 +21,10 @@ pub async fn read_person_connection_test( (location_id, target), ) .await?; - let (transaction, result) = read_location_work(transaction, request_id).await?; - let result = result.map(|stored| stored.record).filter(|record| { - record.request_json.location_id == location_id - && record.request_json.target == target - && matches!(record.request_json.work, LocationWork::CheckConnection {}) - }); + let (transaction, record) = + read_target_work(transaction, (location_id, target, request_id)).await?; + let result = record + .filter(|record| matches!(record.request_json.work, LocationWork::CheckConnection {})); commit(transaction).await?; Ok(result) } diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_person_plan.rs b/crates/sqlbuild-orchestration/src/platform/main/read_person_plan.rs new file mode 100644 index 0000000000..6dd96a14dc --- /dev/null +++ b/crates/sqlbuild-orchestration/src/platform/main/read_person_plan.rs @@ -0,0 +1,65 @@ +//! A plan is read only by the person who asked for it, with view still held on every current node. + +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::credentials::models::{Action, AuditContext, DefaultRole}; +use crate::platform::main::_read_target_work::read_target_work; +use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkRecord}; +use crate::snapshots::main::holds_on_current_nodes::holds_on_current_nodes; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; + +/// Python source and dependency diffs stay out of the plan unless the reader may also read SQL on every current node. +pub async fn read_person_plan( + store: &ControlStore, + (context, principal_id): (&AuditContext, Uuid), + (location_id, target, request_id): (Uuid, &str, Uuid), +) -> Result, ControlError> { + let (transaction, context) = authorize_listing( + begin(store, TransactionMode::Write).await?, + (context, principal_id), + (location_id, target), + DefaultRole::None, + ) + .await?; + let (transaction, record) = + read_target_work(transaction, (location_id, target, request_id)).await?; + let Some(mut record) = record.filter(|record| { + record.requested_by == context.actor + && matches!(record.request_json.work, LocationWork::Plan { .. }) + }) else { + commit(transaction).await?; + return Ok(None); + }; + let (transaction, sql) = holds_on_current_nodes( + transaction, + principal_id, + (Action::ViewSql, location_id, target), + ) + .await?; + commit(transaction).await?; + if !sql { + without_identity_diffs(&mut record); + } + Ok(Some(record)) +} + +fn without_identity_diffs(record: &mut LocationWorkRecord) { + let Some(LocationWorkOutcome::Plan { document }) = record + .result_json + .as_mut() + .map(|result| &mut result.outcome) + else { + return; + }; + let nodes = document + .get_mut("python_nodes") + .and_then(serde_json::Value::as_array_mut); + for node in nodes.into_iter().flatten() { + if let Some(node) = node.as_object_mut() { + node.remove("identity_diff"); + } + } +} diff --git a/crates/sqlbuild-orchestration/src/platform/main/read_person_problems.rs b/crates/sqlbuild-orchestration/src/platform/main/read_person_problems.rs index 5644e4f32c..28b10e4c8c 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/read_person_problems.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/read_person_problems.rs @@ -1,11 +1,11 @@ -//! People see the scheduler's state, and broken locations and late ticks only for targets they list. +//! People see the scheduler's state, and broken locations, late ticks and low calendars only for targets they list. use std::collections::BTreeSet; use uuid::Uuid; -use crate::authorization::main::person_lists_target::person_lists_target; -use crate::credentials::models::DefaultRole; +use crate::authorization::main::person_holds_target::person_holds_target; +use crate::credentials::models::{Action, DefaultRole}; use crate::platform::_helpers::problems::{late_ticks, scan_problems}; use crate::platform::models::PlatformProblems; use crate::store::errors::ControlError; @@ -24,9 +24,21 @@ pub async fn read_person_problems( .iter() .map(|location| (location.location_id, location.target.clone())) .chain(scan.late_targets.iter().cloned()) + .chain( + scan.low_calendars + .iter() + .map(|calendar| (calendar.location_id, calendar.target.clone())), + ) .collect::>() { - if person_lists_target(store, principal_id, (location_id, &target), default_role).await? { + if person_holds_target( + store, + principal_id, + (Action::View, location_id, &target), + default_role, + ) + .await? + { listed.insert((location_id, target)); } } @@ -42,5 +54,10 @@ pub async fn read_person_problems( .into_iter() .filter(|location| listed.contains(&(location.location_id, location.target.clone()))) .collect(), + low_calendars: scan + .low_calendars + .into_iter() + .filter(|calendar| listed.contains(&(calendar.location_id, calendar.target.clone()))) + .collect(), }) } diff --git a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs index 502f521cf8..0e6ca1c7ed 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/request_location_work.rs @@ -1,29 +1,35 @@ //! Named work requires a trusted owner or live person authority; retries preserve the original pin. use chrono::Duration; -use serde_json::json; use uuid::Uuid; -use crate::credentials::main::audit::append_audit::append_audit; use crate::credentials::models::AuditContext; use crate::platform::_helpers::connection_tests::authorize_connection_test; +use crate::platform::main::enqueue_location_work::enqueue_location_work; use crate::platform::main::read_location_work::read_location_work; use crate::platform::models::{LocationWork, LocationWorkRequest}; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; -use crate::store::main::query::query; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::models::{ControlStore, TransactionMode}; pub async fn request_location_work( store: &ControlStore, (context, credentials): (&AuditContext, Option), request: &LocationWorkRequest, ) -> Result { - if matches!(request.work, LocationWork::EvaluateSensor { .. }) { - return Err(ControlError::Invalid( - "sensor work must originate from a fenced evaluation".into(), - )); + match request.work { + LocationWork::EvaluateSensor { .. } => { + return Err(ControlError::Invalid( + "sensor work must originate from a fenced evaluation".into(), + )); + } + LocationWork::Plan { .. } => { + return Err(ControlError::Invalid( + "plans must be requested by a person on the current publication".into(), + )); + } + LocationWork::CheckConnection {} => {} } if request.target.trim().is_empty() || request.reason.trim().is_empty() @@ -50,43 +56,7 @@ pub async fn request_location_work( commit(transaction).await?; return Ok(false); } - let (transaction, rows) = query(transaction, - "SELECT s.snapshot_id, s.generation FROM sqb_current_snapshots c \ - JOIN sqb_snapshots s ON s.snapshot_id=c.snapshot_id JOIN sqb_locations l ON l.location_id=c.location_id \ - WHERE c.location_id=$1 AND c.target=$2 AND l.archived_at IS NULL", - vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone())]).await?; - let snapshot = rows - .first() - .ok_or_else(|| ControlError::Invalid("location work requires a published target".into()))?; - let snapshot_id = serde_json::from_value(snapshot["snapshot_id"].clone()) - .map_err(|error| ControlError::Invalid(format!("invalid snapshot identity: {error}")))?; - let generation = snapshot["generation"] - .as_i64() - .ok_or_else(|| ControlError::Invalid("invalid snapshot generation".into()))?; - let now = transaction.now; - let (transaction, _) = query(transaction, - "INSERT INTO sqb_location_work \ - (request_id,location_id,target,snapshot_id,generation,request_json,requested_by,requested_at,expires_at) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", - vec![Parameter::Id(request.request_id), Parameter::Id(request.location_id), Parameter::Text(request.target.clone()), - Parameter::Id(snapshot_id), Parameter::Integer(generation), Parameter::Json(json!(request)), - Parameter::Text(context.actor.clone()), Parameter::Time(now), Parameter::Time(now + Duration::minutes(5))]).await?; - commit( - append_audit( - transaction, - &context, - ( - Uuid::new_v5(&request.request_id, b"location-work-requested"), - "location.work_requested", - ), - ( - json!({"location_id": request.location_id, "target": request.target}), - json!(null), - json!(request), - ), - ) - .await?, - ) - .await?; + commit(enqueue_location_work(transaction, &context, request, Duration::minutes(5)).await?) + .await?; Ok(true) } diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 1bb5c963c8..928081c82e 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -73,6 +73,13 @@ pub enum LocationWork { #[schema(value_type = Object)] evaluation: Box, }, + /// `sqb plan --json` on exact node keys the server froze from a person's build command. + Plan { + command: Vec, + selection: std::collections::BTreeSet, + #[schema(value_type = Object)] + settings: std::collections::BTreeMap, + }, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] @@ -85,6 +92,14 @@ pub struct LocationWorkRequest { pub work: LocationWork, } +/// A person's build command to plan on the location; the server freezes it like an ad-hoc run. +#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct PlanRequest { + pub request_id: Uuid, + pub command: Vec, +} + #[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct ConnectionTestRequest { @@ -112,8 +127,17 @@ pub struct LocationWorkClaim { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum LocationWorkOutcome { - Connection { healthy: bool }, - SensorEvaluation { outcome: CustomSensorOutcome }, + Connection { + healthy: bool, + }, + SensorEvaluation { + outcome: CustomSensorOutcome, + }, + /// The plan document `sqb plan --json` printed, or its `{"error": ...}` report. + Plan { + #[schema(value_type = Object)] + document: serde_json::Value, + }, Superseded {}, } @@ -159,7 +183,7 @@ pub enum PlatformEventKind { LaunchFailed, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum PlatformEvent { Heartbeat { @@ -237,6 +261,33 @@ pub(crate) struct WatchdogLocation { pub(crate) last_seen_at: DateTime, } +/// A stored platform event a person may see; the location target is absent for server-wide events. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct PersonPlatformEvent { + pub storage_id: i64, + pub event_id: Uuid, + pub occurred_at: DateTime, + pub event: PlatformEvent, + pub location_id: Option, + pub target: Option, +} + +/// One scan of stored platform events, keeping those the person may see. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct PersonPlatformEventPage { + pub events: Vec, + /// The `before` that continues the scan, absent once no older events remain; a page can be empty and still continue. + pub before: Option, +} + +/// The location target a trigger or run belongs to. +#[derive(Debug, Deserialize)] +pub(crate) struct EventScope { + pub(crate) id: Uuid, + pub(crate) location_id: Uuid, + pub(crate) target: String, +} + /// Stable event identity permits receiver-side deduplication after uncertain acknowledgements. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct PlatformEventEnvelope { @@ -259,6 +310,34 @@ pub struct PlatformProblems { pub scheduler_lease_expires_at: Option>, pub broken_locations: Vec, pub overdue_ticks: Vec, + /// Running calendar schedules whose dates run out within a day. + #[serde(default)] + pub low_calendars: Vec, +} + +/// A running calendar schedule whose last date is at most a day away, or which has none left. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct LowCalendar { + pub trigger_id: Uuid, + pub name: String, + pub location_id: Uuid, + pub location_name: String, + pub target: String, + /// When the location next works out the calendar's dates. + pub refresh_at: DateTime, + /// The calendar's last date; absent when none are left. + pub horizon_end: Option>, +} + +/// A running schedule of a live location, with its declaration. +#[derive(Debug, Deserialize)] +pub(crate) struct RunningSchedule { + pub(crate) trigger_id: Uuid, + pub(crate) name: String, + pub(crate) location_id: Uuid, + pub(crate) location_name: String, + pub(crate) target: String, + pub(crate) definition_json: TriggerDefinition, } /// Every instance of this location target has missed two check-ins. @@ -299,4 +378,31 @@ pub(crate) struct ProblemScan { pub(crate) lease: Option, pub(crate) broken: Vec, pub(crate) late_targets: std::collections::BTreeSet<(Uuid, String)>, + pub(crate) low_calendars: Vec, +} + +/// Where the control store keeps its data. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum StorageKind { + Sqlite, + Postgresql, +} + +/// The control store's installation and schema, as recorded when it was created. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct InstallationRecord { + pub storage: StorageKind, + pub installed_at: DateTime, + /// Digest of the schema; a server starts only on the exact schema it creates. + pub schema_checksum: String, + pub schema_applied_at: DateTime, +} + +/// The stored installation and schema rows. +#[derive(Debug, Deserialize)] +pub(crate) struct StoredInstallation { + pub(crate) installed_at: DateTime, + pub(crate) schema_checksum: String, + pub(crate) schema_applied_at: DateTime, } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs index 6b7c17d485..6a711597c2 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs @@ -92,11 +92,40 @@ pub(crate) fn earlier_waiter_blocks( limits: &BTreeSet, ) -> bool { let position = rank(candidate); - waiters.iter().any(|waiter| { - waiter.run_id != candidate.request_json.run_id - && waiter.rank < position - && !waiter.limits.is_disjoint(limits) - }) + waiters + .iter() + .any(|waiter| ahead_on(waiter, (candidate, position), limits).is_some()) +} + +/// Waiters ranked ahead of the candidate on a limit it takes, each with the first limit they share. +pub(crate) fn earlier_waiters<'a>( + waiters: &'a [LimitWaiter], + candidate: &QueueCandidate, + limits: &'a BTreeSet, +) -> Vec<(&'a LimitWaiter, &'a String)> { + let position = rank(candidate); + let mut earlier: Vec<(&LimitWaiter, &String)> = Vec::new(); + for waiter in waiters { + if let Some(limit) = ahead_on(waiter, (candidate, position), limits) { + earlier.push((waiter, limit)); + } + } + earlier +} + +/// The first of `limits` the waiter shares when it ranks ahead of the candidate. +fn ahead_on<'a>( + waiter: &LimitWaiter, + (candidate, position): ( + &QueueCandidate, + (Reverse, chrono::DateTime, Uuid), + ), + limits: &'a BTreeSet, +) -> Option<&'a String> { + if waiter.run_id == candidate.request_json.run_id || waiter.rank >= position { + return None; + } + limits.iter().find(|limit| waiter.limits.contains(*limit)) } fn rank(run: &QueueCandidate) -> (Reverse, chrono::DateTime, Uuid) { diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs index a4a6f16c6c..471f2e7d52 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/selection.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::runs::_helpers::groups::context::load_group; -use crate::runs::models::{RunRequest, RunStatus}; +use crate::runs::models::{QueueWait, RunRequest, RunStatus}; use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; use crate::store::main::query::query; @@ -13,8 +13,17 @@ pub(crate) async fn child_ready( transaction: ControlTransaction, request: &RunRequest, ) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, waits) = child_waits(transaction, request).await?; + Ok((transaction, waits.is_empty())) +} + +/// What holds a group child back: its group not running, or upstream jobs of the group not finished. +pub(crate) async fn child_waits( + transaction: ControlTransaction, + request: &RunRequest, +) -> Result<(ControlTransaction, Vec), ControlError> { let Some(parent_id) = request.parent_run_id else { - return Ok((transaction, true)); + return Ok((transaction, Vec::new())); }; let (transaction, group) = load_group(transaction, parent_id).await?; let name = request @@ -28,18 +37,29 @@ pub(crate) async fn child_ready( .as_ref() .ok_or_else(|| ControlError::Invalid("group has no declaration".into()))?; let dependencies = upstream_jobs(&group.definitions.groups[parent_name].jobs, name); - let ready = group.parent.state.status == RunStatus::Running - && group.parent.state.cancel_requested_at.is_none() - && !group.stopped() - && group.children.iter().all(|child| { - child - .request_json - .job_key - .as_ref() - .is_none_or(|name| !dependencies.contains(name.as_str())) - || child.state.status.is_terminal() - }); - Ok((transaction, ready)) + if group.parent.state.status != RunStatus::Running + || group.parent.state.cancel_requested_at.is_some() + || group.stopped() + { + return Ok(( + transaction, + vec![QueueWait::GroupParent { run_id: parent_id }], + )); + } + let waits: Vec = group + .children + .iter() + .filter_map(|child| { + let job_key = child.request_json.job_key.as_ref()?; + (dependencies.contains(job_key.as_str()) && !child.state.status.is_terminal()).then( + || QueueWait::UpstreamJob { + run_id: child.request_json.run_id, + job_key: job_key.clone(), + }, + ) + }) + .collect(); + Ok((transaction, waits)) } pub(crate) async fn skipped_nodes( diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs index 1c4f2e737c..4b79246a14 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -4,6 +4,7 @@ pub(in crate::runs) mod checks; pub(in crate::runs) mod claims; pub(crate) mod error_text; pub(in crate::runs) mod events; +pub(crate) mod narrowing; pub(in crate::runs) mod nodes; pub(in crate::runs) mod output; pub(in crate::runs) mod projection; @@ -11,3 +12,4 @@ pub(in crate::runs) mod publish_change; pub(in crate::runs) mod queued; pub(in crate::runs) mod reads; pub(in crate::runs) mod recall; +pub(in crate::runs) mod waits; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/narrowing.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/narrowing.rs new file mode 100644 index 0000000000..ae1b5511ef --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/narrowing.rs @@ -0,0 +1,55 @@ +//! A run's narrowing and backfill window are read from its frozen request, and nothing else of that request is returned. + +use serde::{Deserialize, Deserializer}; +use serde_json::Value; + +use crate::runs::models::{RunNarrowing, RunWindow}; + +/// The narrowing of a stored frozen request, which reruns copy, or of an already projected record. +pub(crate) fn frozen_narrowing<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let value: Value = frozen(deserializer)?; + let fields: &Value = value.get("narrowing").unwrap_or(&value); + let tokens = |name: &str| -> Vec { + fields + .get(name) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|token| token.as_str().map(str::to_owned)) + .collect() + }; + let narrowing = RunNarrowing { + select: tokens("select"), + exclude: tokens("exclude"), + }; + Ok((!narrowing.select.is_empty() || !narrowing.exclude.is_empty()).then_some(narrowing)) +} + +/// The backfill window of a stored frozen request, or of an already projected record: a stored request always has its run ID, and a projected window never does. +pub(crate) fn frozen_window<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let value: Value = frozen(deserializer)?; + let window: &Value = match value.get("window") { + Some(window) => window, + None if value.get("run_id").is_none() => &value, + None => return Ok(None), + }; + if window.is_null() { + return Ok(None); + } + serde_json::from_value(window.clone()) + .map(Some) + .map_err(serde::de::Error::custom) +} + +/// A JSON column, which SQLite returns as text and PostgreSQL as JSON. +fn frozen<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let value: Value = Option::::deserialize(deserializer)?.unwrap_or(Value::Null); + Ok(match value { + Value::String(text) => serde_json::from_str(&text).unwrap_or(Value::Null), + other => other, + }) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs index bddbdfa2df..f1f686b3fc 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs @@ -2,25 +2,53 @@ use uuid::Uuid; +use crate::runs::_helpers::lifecycle::waits::queue_states; + use crate::runs::models::{RunNodeRecord, RunRecord}; +use crate::runs::types::RunsListing; use crate::store::errors::ControlError; use crate::store::main::query_records::query_records; use crate::store::models::{ControlTransaction, Parameter}; +/// Newest first after an optional cursor, or the queue in claim order with each run's queue state. pub(crate) async fn list_runs_in( transaction: ControlTransaction, (location_id, target): (Uuid, &str), - limit: u32, + listing: RunsListing, ) -> Result<(ControlTransaction, Vec), ControlError> { - query_records(transaction, + let (limit, before) = match listing { + RunsListing::Newest(limit, before) => (limit, before), + RunsListing::Queue(limit) => (limit, None), + }; + let mut parameters = vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Integer(i64::from(limit.min(1000))), + ]; + let filter = match (listing, before) { + (RunsListing::Queue(_), _) => { + "AND r.status='queued' AND r.worker_instance IS NULL ORDER BY r.priority DESC, r.queued_at, r.run_id" + } + (_, Some((queued_at, run_id))) => { + parameters.extend([Parameter::Time(queued_at), Parameter::Id(run_id)]); + "AND (r.queued_at<$4 OR (r.queued_at=$4 AND r.run_id<$5)) ORDER BY r.queued_at DESC, r.run_id DESC" + } + (_, None) => "ORDER BY r.queued_at DESC, r.run_id DESC", + }; + let (transaction, mut runs) = query_records(transaction, &format!( "SELECT r.run_id, r.location_id, r.target, r.snapshot_id, r.job_key, r.kind, r.origin, r.parent_run_id, r.root_run_id, \ r.trigger_id, t.name AS trigger_name, r.tick_id, r.rerun_of_run_id, r.requested_by, p.name AS requester_name, r.priority, \ r.rewrite_history, r.status, r.status_reason, r.queued_at, r.launched_at, r.started_at, r.ended_at, r.cancel_requested_at, \ - r.selection_json, r.node_counts_json, r.last_event_storage_id \ + r.selection_json, r.node_counts_json, r.last_event_storage_id, r.request_json AS narrowing, r.request_json AS backfill_window \ FROM sqb_runs r LEFT JOIN sqb_triggers t ON t.trigger_id=r.trigger_id \ LEFT JOIN sqb_principals p ON 'principal:' || CAST(p.principal_id AS TEXT)=r.requested_by \ - WHERE r.location_id=$1 AND r.target=$2 ORDER BY r.queued_at DESC, r.run_id DESC LIMIT $3", - vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(i64::from(limit.min(1000)))]).await + WHERE r.location_id=$1 AND r.target=$2 {filter} LIMIT $3"), + parameters).await?; + if !matches!(listing, RunsListing::Queue(_)) { + return Ok((transaction, runs)); + } + let transaction = queue_states(transaction, (location_id, target), &mut runs).await?; + Ok((transaction, runs)) } pub(crate) async fn run_nodes_in( diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/waits.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/waits.rs new file mode 100644 index 0000000000..1dac6910b6 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/waits.rs @@ -0,0 +1,154 @@ +//! Why queued runs wait, read with the admission rules the scheduler claims by; it reserves nothing. + +use std::collections::{BTreeMap, BTreeSet}; + +use uuid::Uuid; + +use crate::runs::_helpers::groups::admission::{earlier_waiters, limit_waiters}; +use crate::runs::_helpers::groups::selection::child_waits; +use crate::runs::models::{ + LimitWaiter, QueueCandidate, QueueHolder, QueueWait, RunQueueState, RunRecord, RunningJob, +}; +use crate::store::errors::ControlError; +use crate::store::main::{decode_row::decode_row, query::query}; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::types::Row; + +/// Set each run's position and waits; `runs` are this scope's queued runs in claim order. +pub(crate) async fn queue_states( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + runs: &mut [RunRecord], +) -> Result { + let scope = vec![Parameter::Id(location_id), Parameter::Text(target.into())]; + let (transaction, waiters) = limit_waiters(transaction, None).await?; + let (transaction, rows) = query(transaction, + "SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.worker_instance IS NULL AND r.location_id=$1 AND r.target=$2", + scope.clone()).await?; + let candidates: BTreeMap = decoded::(rows)? + .into_iter() + .map(|run| (run.request_json.run_id, run)) + .collect(); + let (transaction, rows) = query(transaction, + "SELECT w.node_key AS held, w.run_id, r.location_id, r.target FROM sqb_write_claims w JOIN sqb_runs r ON r.run_id=w.run_id WHERE w.target=$1", + vec![Parameter::Text(target.into())]).await?; + let claims: Vec = decoded(rows)?; + let (transaction, rows) = query(transaction, + "SELECT s.group_key AS held, s.run_id, r.location_id, r.target FROM sqb_concurrency_slots s JOIN sqb_runs r ON r.run_id=s.run_id", + vec![]).await?; + let slots: Vec = decoded(rows)?; + let (mut transaction, rows) = query(transaction, + "SELECT r.run_id, r.job_key FROM sqb_runs r WHERE r.location_id=$1 AND r.target=$2 AND r.job_key IS NOT NULL \ + AND (r.worker_instance IS NOT NULL OR (r.kind='group' AND r.status='running'))", + scope).await?; + let running: Vec = decoded(rows)?; + let shown = |holder: &QueueHolder| { + (holder.location_id == location_id && holder.target == target).then_some(holder.run_id) + }; + for (position, run) in (1..).zip(runs.iter_mut()) { + let Some(candidate) = candidates.get(&run.run_id) else { + continue; + }; + let request = &candidate.request_json; + let mut waits; + (transaction, waits) = child_waits(transaction, request).await?; + waits.extend( + running + .iter() + .filter(|job| { + job.run_id != request.run_id + && request.job_key.as_deref() == Some(job.job_key.as_str()) + }) + .map(|job| QueueWait::JobRunning { run_id: job.run_id }), + ); + waits.extend(write_claims( + &claims, + &shown, + (request.run_id, &request.write_set), + )); + let limits: BTreeSet = waiters + .iter() + .find(|waiter| waiter.run_id == request.run_id) + .map(|waiter| waiter.limits.clone()) + .unwrap_or_default(); + waits.extend(held_limits(&slots, &shown, (request.run_id, &limits))); + waits.extend(earlier_runs(&waiters, (candidate, &limits), &candidates)); + run.queue = Some(RunQueueState { position, waits }); + } + Ok(transaction) +} + +fn decoded(rows: Vec) -> Result, ControlError> { + rows.into_iter().map(decode_row).collect() +} + +/// One wait per other run holding nodes this run writes, naming its first node. +fn write_claims( + claims: &[QueueHolder], + shown: &impl Fn(&QueueHolder) -> Option, + (run_id, write_set): (Uuid, &BTreeSet), +) -> Vec { + let mut held: BTreeMap<(Option, Uuid), Vec<&str>> = BTreeMap::new(); + for claim in claims { + if claim.run_id != run_id && write_set.contains(&claim.held) { + held.entry((shown(claim), claim.run_id)) + .or_default() + .push(&claim.held); + } + } + held.into_iter() + .map(|((shown, _), nodes)| QueueWait::WriteClaim { + run_id: shown, + node_key: nodes[0].to_owned(), + other_nodes: count(nodes.len() - 1), + }) + .collect() +} + +/// Each limit this run takes that running runs hold, naming holders in this scope and counting the rest. +fn held_limits( + slots: &[QueueHolder], + shown: &impl Fn(&QueueHolder) -> Option, + (run_id, limits): (Uuid, &BTreeSet), +) -> Vec { + let mut waits: Vec = Vec::new(); + for limit in limits { + let held: Vec<&QueueHolder> = slots + .iter() + .filter(|slot| &slot.held == limit && slot.run_id != run_id) + .collect(); + if held.is_empty() { + continue; + } + let holders: Vec = held.iter().filter_map(|slot| shown(slot)).collect(); + waits.push(QueueWait::ConcurrencyLimit { + limit: limit.clone(), + elsewhere: count(held.len() - holders.len()), + holders, + }); + } + waits +} + +/// The first earlier waiter on each shared limit; one queued outside this scope is not named. +fn earlier_runs( + waiters: &[LimitWaiter], + (candidate, limits): (&QueueCandidate, &BTreeSet), + here: &BTreeMap, +) -> Vec { + let mut ahead: BTreeSet<&String> = BTreeSet::new(); + let mut waits: Vec = Vec::new(); + for (waiter, limit) in earlier_waiters(waiters, candidate, limits) { + if ahead.insert(limit) { + waits.push(QueueWait::EarlierRun { + run_id: here.contains_key(&waiter.run_id).then_some(waiter.run_id), + limit: limit.clone(), + }); + } + } + waits +} + +fn count(value: usize) -> u32 { + u32::try_from(value).unwrap_or(u32::MAX) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index 3fb6c0739b..137c35efee 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -8,17 +8,18 @@ use uuid::Uuid; use crate::authorization::main::authorize_listing::authorize_listing; use crate::authorization::main::decide_person_scopes::decide_person_scopes; use crate::credentials::main::audit::append_audit::append_audit; -use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; +use crate::credentials::models::{Action, AuditContext, DefaultRole}; +use crate::declarations::main::frozen_definitions::frozen_definitions; use crate::runs::_helpers::requests::permissions::{authorize_execution, execution_scopes}; use crate::runs::_helpers::requests::publication::{ publish_request, record_authored, request_retry, }; use crate::runs::constants::ADHOC_AUDIT_TAG; use crate::runs::models::{ - AdhocCheckRequest, AdhocPermission, AdhocRunRequest, PlannerResource, RunCommand, RunKind, - RunOrigin, RunRequest, + AdhocCheckRequest, AdhocPermission, AdhocRunRequest, CommandPlan, PlannerResource, RunCommand, + RunKind, RunOrigin, RunRequest, }; -use crate::runs::types::CommandPlanner; +use crate::runs::types::{CommandPlanner, PublishedJobs}; use crate::snapshots::main::job_write_set::job_write_set; use crate::snapshots::main::read_current_record::read_current_record; use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; @@ -44,8 +45,13 @@ pub(crate) async fn frozen_adhoc_request( Err("this target has no publication yet".into()), )); }; - let (transaction, nodes) = read_snapshot_nodes(transaction, snapshot.snapshot_id, None).await?; - let plan = match planner.plan(&request.command, &planner_resources(&nodes)) { + let (transaction, plan) = publication_plan( + transaction, + (planner, snapshot.snapshot_id), + &request.command, + ) + .await?; + let plan = match plan { Ok(plan) => plan, Err(refusal) => return Ok((transaction, Err(refusal))), }; @@ -99,11 +105,37 @@ pub(crate) async fn frozen_adhoc_request( settings: plan.settings, window: None, concurrency_groups: BTreeSet::new(), + narrowing: None, }, )), )) } +/// `command` planned over one publication's nodes and its jobs, or why the planner refuses it. +pub(crate) async fn publication_plan( + transaction: ControlTransaction, + (planner, snapshot_id): (&dyn CommandPlanner, Uuid), + command: &[String], +) -> Result<(ControlTransaction, Result), ControlError> { + let (transaction, nodes) = read_snapshot_nodes(transaction, snapshot_id, None).await?; + let (transaction, definitions) = frozen_definitions(transaction, snapshot_id).await?; + let jobs: PublishedJobs = definitions + .jobs + .into_iter() + .map(|(name, job)| (name, job.selection)) + .chain( + definitions + .groups + .into_iter() + .map(|(name, group)| (name, group.job.selection)), + ) + .collect(); + Ok(( + transaction, + planner.plan(command, (&planner_resources(&nodes), &jobs)), + )) +} + /// The selector graph rebuilt from published definitions, in the CLI's kind order. fn planner_resources( nodes: &std::collections::BTreeMap, @@ -154,30 +186,6 @@ fn texts(value: Option<&Value>) -> Vec { .collect() } -/// Whether the person may view every current node, decided without recording a denial. -pub(in crate::runs) async fn may_view_target( - transaction: ControlTransaction, - principal_id: Uuid, - request: &AdhocCheckRequest, -) -> Result<(ControlTransaction, bool), ControlError> { - let (transaction, current) = query(transaction, - "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_current_snapshots c ON c.snapshot_id=n.snapshot_id \ - WHERE c.location_id=$1 AND c.target=$2", - vec![Parameter::Id(request.location_id), Parameter::Text(request.target.clone())]).await?; - let view = PermissionScope { - action: Action::View, - location_id: Some(request.location_id), - target: Some(request.target.clone()), - nodes: current - .iter() - .filter_map(|row| row["node_key"].as_str().map(str::to_owned)) - .collect(), - }; - let (transaction, refusals) = - decide_person_scopes(transaction, principal_id, (&request.target, &[view])).await?; - Ok((transaction, refusals.as_deref() == Some(&[None]))) -} - /// Every permission launching `frozen` needs, decided as the launch would but without recording a denial. pub(in crate::runs) async fn permission_decisions( transaction: ControlTransaction, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs index 6e83f5519c..19c55a45f3 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs @@ -3,6 +3,7 @@ use serde_json::json; use uuid::Uuid; +use crate::authorization::main::authorize_listing::authorize_listing; use crate::authorization::main::authorize_person::authorize_person; use crate::authorization::main::refuse_unknown::refuse_unknown; use crate::credentials::main::audit::append_audit::append_audit; @@ -14,14 +15,16 @@ use crate::runs::_helpers::requests::publication::{ use crate::runs::constants::MANUAL_JOB_AUDIT_TAG; use crate::runs::main::frozen_job_request::frozen_job_request; use crate::runs::models::{JobRunRequest, RunWindow}; +use crate::runs::types::CommandPlanner; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; +/// Narrowing resolves selectors over the whole publication, so a person needs view on every node to narrow. pub(crate) async fn request_job_run_in( mut transaction: ControlTransaction, (context, credentials): (&AuditContext, Option), - request: &JobRunRequest, + (planner, request): (Option<&dyn CommandPlanner>, &JobRunRequest), ) -> Result<(ControlTransaction, bool), ControlError> { let mut context = context.clone(); let scope = PermissionScope { @@ -38,6 +41,15 @@ pub(crate) async fn request_job_run_in( DefaultRole::None, ) .await?; + if !request.select.is_empty() || !request.exclude.is_empty() { + (transaction, _) = authorize_listing( + transaction, + (&context, credentials), + (request.location_id, &request.target), + DefaultRole::None, + ) + .await?; + } } if context.actor.trim().is_empty() || context.granting_role.trim().is_empty() @@ -72,7 +84,8 @@ pub(crate) async fn request_job_run_in( Some(credentials) => known_job(transaction, (&context, credentials), request).await?, None => transaction, }; - let (mut transaction, frozen) = frozen_job_request(transaction, &context, request).await?; + let (mut transaction, frozen) = + frozen_job_request(transaction, (&context, planner), request).await?; if let Some(credentials) = credentials { (transaction, context) = authorize_execution( transaction, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs index 2231ab538f..85c7ebfc6d 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs @@ -4,5 +4,6 @@ pub(in crate::runs) mod job; pub(in crate::runs) mod manual_job; pub(in crate::runs) mod manual_rerun; pub(in crate::runs) mod permissions; +pub(in crate::runs) mod plan; pub(in crate::runs) mod publication; pub(in crate::runs) mod rerun; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs new file mode 100644 index 0000000000..89e1c0bbee --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs @@ -0,0 +1,116 @@ +//! A plan is frozen like an ad-hoc build but queued as location work, so it never becomes a run. + +use chrono::Duration; +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::platform::constants::{LOCATION_SILENCE_SECONDS, PLAN_EXPIRY_SECONDS, PLAN_QUEUE_LIMIT}; +use crate::platform::main::enqueue_location_work::enqueue_location_work; +use crate::platform::main::read_location_work::read_location_work; +use crate::platform::models::{LocationWork, LocationWorkRequest, PlanRequest}; +use crate::runs::_helpers::requests::adhoc::frozen_adhoc_request; +use crate::runs::constants::{PLAN_IGNORED_SETTING, PLAN_REASON}; +use crate::runs::models::{AdhocCheckRequest, RunCommand}; +use crate::runs::types::CommandPlanner; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Needs view on every current node; a refused command, a silent location or a full queue queues nothing. +pub(in crate::runs) async fn request_plan_in( + transaction: ControlTransaction, + (context, principal_id, planner): (&AuditContext, Uuid, &dyn CommandPlanner), + (location_id, target, request): (Uuid, &str, &PlanRequest), +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, context) = authorize_listing( + transaction, + (context, principal_id), + (location_id, target), + DefaultRole::None, + ) + .await?; + let (transaction, previous) = read_location_work(transaction, request.request_id).await?; + if let Some(previous) = previous { + let same = previous.record.requested_by == context.actor + && previous.record.request_json.location_id == location_id + && previous.record.request_json.target == target + && matches!(&previous.record.request_json.work, + LocationWork::Plan { command, .. } if *command == request.command); + if !same { + return Err(ControlError::Invalid( + "location work ID already has different content or actor".into(), + )); + } + return Ok((transaction, false)); + } + let check = AdhocCheckRequest { + location_id, + target: target.to_owned(), + command: request.command.clone(), + priority: 0, + }; + let (transaction, frozen) = frozen_adhoc_request( + transaction, + (&context, planner), + (&check, request.request_id), + ) + .await?; + let (verb, frozen) = frozen.map_err(ControlError::Invalid)?; + if frozen.command != RunCommand::Build { + return Err(ControlError::Invalid(format!( + "sqb plan previews builds; sqb {verb} cannot be planned" + ))); + } + let now = transaction.now; + let (transaction, rows) = query(transaction, + "SELECT \ + (SELECT COUNT(*) FROM sqb_location_instances i WHERE i.location_id=$1 AND i.target=$2 AND i.last_seen_at>=$3) AS live, \ + (SELECT COUNT(*) FROM sqb_location_work w WHERE w.location_id=$1 AND w.target=$2 \ + AND w.completed_at IS NULL AND w.expires_at>$4 AND w.request_json->'work'->>'kind'='plan') AS queued, \ + (SELECT COUNT(*) FROM sqb_location_work w WHERE w.location_id=$1 AND w.target=$2 \ + AND w.completed_at IS NULL AND w.expires_at>$4 AND w.request_json->'work'->>'kind'='plan' \ + AND w.requested_by=$5) AS mine", + vec![Parameter::Id(location_id), Parameter::Text(target.to_owned()), + Parameter::Time(now - Duration::seconds(LOCATION_SILENCE_SECONDS)), Parameter::Time(now), + Parameter::Text(context.actor.clone())]).await?; + let count = |name: &str| rows.first().and_then(|row| row[name].as_i64()).unwrap_or(0); + if count("live") == 0 { + return Err(ControlError::Invalid(format!( + "no sqb location serve has reported for {target} in the last {LOCATION_SILENCE_SECONDS} seconds; start it and plan again" + ))); + } + if count("mine") > 0 { + return Err(ControlError::Invalid( + "your previous plan for this target is still running".into(), + )); + } + if count("queued") >= PLAN_QUEUE_LIMIT { + return Err(ControlError::Invalid(format!( + "{PLAN_QUEUE_LIMIT} plans are already waiting for this target; try again shortly" + ))); + } + let work = LocationWorkRequest { + request_id: request.request_id, + location_id, + target: target.to_owned(), + reason: PLAN_REASON.into(), + work: LocationWork::Plan { + command: request.command.clone(), + selection: frozen.selection, + settings: frozen + .settings + .into_iter() + .filter(|(name, _)| name != PLAN_IGNORED_SETTING) + .collect(), + }, + }; + let transaction = enqueue_location_work( + transaction, + &context, + &work, + Duration::seconds(PLAN_EXPIRY_SECONDS), + ) + .await?; + Ok((transaction, true)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs index cad5c75e1f..30ae961508 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs @@ -1,10 +1,11 @@ -//! Retry immutable execution inputs, narrowing only to durable failed and skipped outcomes. +//! Retry immutable execution inputs, narrowing only to durable failed and skipped outcomes or to named nodes of the source run. use std::collections::BTreeSet; use serde_json::from_value; use uuid::Uuid; +use crate::authorization::main::authorize_listing::authorize_listing; use crate::authorization::main::authorize_person::authorize_person; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::runs::_helpers::requests::permissions::authorize_execution; @@ -40,6 +41,27 @@ pub(crate) async fn frozen_rerun( .map_err(|error| ControlError::Invalid(error.to_string()))?; let mut frozen: RunRequest = from_value(source["request_json"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?; + let named: BTreeSet<&str> = request.nodes.iter().map(String::as_str).collect(); + if !named.is_empty() { + if request.from_failure { + return Err(ControlError::Invalid( + "rerun either named nodes or from failure".into(), + )); + } + if let Some(credentials) = credentials { + (transaction, _) = authorize_listing( + transaction, + (context, credentials), + (frozen.location_id, &frozen.target), + DefaultRole::None, + ) + .await?; + } + frozen.selection.retain(|key| named.contains(key.as_str())); + frozen + .write_set + .retain(|key| frozen.selection.contains(key)); + } if request.from_failure { let rows; (transaction, rows) = query( @@ -86,5 +108,10 @@ pub(crate) async fn frozen_rerun( "source run has no failed or skipped nodes".into(), )); } + if frozen.selection.len() != named.len() && !named.is_empty() { + return Err(ControlError::Invalid( + "rerun nodes must be in the source run's selection".into(), + )); + } Ok((transaction, (context, frozen))) } diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index 9a7bea5f08..6842fddad0 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -20,3 +20,9 @@ pub(crate) const ADHOC_AUDIT_TAG: &[u8] = b"adhoc-run-request"; /// The build setting a full-refresh launch adds to the frozen request and each group child. pub(crate) const FULL_REFRESH_SETTING: &str = "full_refresh"; + +/// The reason recorded on a person's plan request; plans carry no reason of their own. +pub(crate) const PLAN_REASON: &str = "plan"; + +/// A plan only reads warehouse metadata, so it never takes a warehouse a viewer names; `run.warehouse` gates that for runs. +pub(crate) const PLAN_IGNORED_SETTING: &str = "warehouse"; diff --git a/crates/sqlbuild-orchestration/src/runs/main/check_adhoc_run.rs b/crates/sqlbuild-orchestration/src/runs/main/check_adhoc_run.rs index 4874d10070..44ede8a7fc 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/check_adhoc_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/check_adhoc_run.rs @@ -2,12 +2,11 @@ use uuid::Uuid; -use crate::credentials::models::AuditContext; -use crate::runs::_helpers::requests::adhoc::{ - frozen_adhoc_request, may_view_target, permission_decisions, -}; +use crate::credentials::models::{Action, AuditContext}; +use crate::runs::_helpers::requests::adhoc::{frozen_adhoc_request, permission_decisions}; use crate::runs::models::{AdhocCheck, AdhocCheckRequest}; use crate::runs::types::CommandPlanner; +use crate::snapshots::main::holds_on_current_nodes::holds_on_current_nodes; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; use crate::store::models::{ControlStore, TransactionMode}; @@ -20,7 +19,12 @@ pub async fn check_adhoc_run( request: &AdhocCheckRequest, ) -> Result { let transaction = begin(store, TransactionMode::Read).await?; - let (transaction, viewing) = may_view_target(transaction, principal_id, request).await?; + let (transaction, viewing) = holds_on_current_nodes( + transaction, + principal_id, + (Action::View, request.location_id, &request.target), + ) + .await?; let (transaction, frozen) = if viewing { frozen_adhoc_request(transaction, (context, planner), (request, Uuid::nil())).await? } else { diff --git a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs index 85e8a26843..dd3ca6fc2f 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs @@ -1,17 +1,26 @@ //! Freeze published jobs for manual and sensor requests in their owning transaction. +use std::collections::BTreeSet; + +use uuid::Uuid; + use crate::credentials::models::AuditContext; use crate::declarations::models::JobDefinition; +use crate::runs::_helpers::requests::adhoc::publication_plan; use crate::runs::constants::FULL_REFRESH_SETTING; -use crate::runs::models::{JobRunRequest, RunCommand, RunKind, RunOrigin, RunRequest}; +use crate::runs::models::{ + JobRunRequest, RunCommand, RunKind, RunNarrowing, RunOrigin, RunRequest, +}; +use crate::runs::types::CommandPlanner; use crate::snapshots::main::job_write_set::job_write_set; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; +/// Without a planner, a request that narrows the job is refused. pub(crate) async fn frozen_job_request( transaction: ControlTransaction, - context: &AuditContext, + (context, planner): (&AuditContext, Option<&dyn CommandPlanner>), request: &JobRunRequest, ) -> Result<(ControlTransaction, RunRequest), ControlError> { let (transaction, jobs) = query( @@ -37,6 +46,20 @@ pub(crate) async fn frozen_job_request( .settings .insert(FULL_REFRESH_SETTING.into(), serde_json::Value::Bool(true)); } + let mut transaction = transaction; + let narrowing: Option = + (!request.select.is_empty() || !request.exclude.is_empty()).then(|| RunNarrowing { + select: request.select.clone(), + exclude: request.exclude.clone(), + }); + if narrowing.is_some() { + (transaction, definition.selection) = narrowed( + transaction, + (planner, snapshot_id, kind), + (request, definition.selection), + ) + .await?; + } let (transaction, write_set) = job_write_set(transaction, snapshot_id, &definition.selection).await?; Ok(( @@ -71,6 +94,48 @@ pub(crate) async fn frozen_job_request( settings: definition.settings, window: request.window.clone(), concurrency_groups: definition.concurrency_groups, + narrowing, }, )) } + +/// The job's nodes that `sqb build --select … --exclude …` also selects; never a node outside the job. +async fn narrowed( + transaction: ControlTransaction, + (planner, snapshot_id, kind): (Option<&dyn CommandPlanner>, Uuid, RunKind), + (request, selection): (&JobRunRequest, BTreeSet), +) -> Result<(ControlTransaction, BTreeSet), ControlError> { + let invalid = |message: &str| Err(ControlError::Invalid(message.into())); + if kind == RunKind::Group { + return invalid("a job group cannot be narrowed; narrow one of its jobs"); + } + let Some(planner) = planner else { + return invalid("this request cannot narrow a job"); + }; + if request + .select + .iter() + .chain(&request.exclude) + .any(|token| token.trim().is_empty() || token.starts_with('-')) + { + return invalid("job selectors must be selector tokens, not flags"); + } + let mut command: Vec = vec!["build".into()]; + for (flag, tokens) in [ + ("--select", &request.select), + ("--exclude", &request.exclude), + ] { + if !tokens.is_empty() { + command.push(flag.into()); + command.extend(tokens.iter().cloned()); + } + } + let (transaction, plan) = + publication_plan(transaction, (planner, snapshot_id), &command).await?; + let plan = plan.map_err(ControlError::Invalid)?; + let narrowed: BTreeSet = selection.intersection(&plan.selection).cloned().collect(); + if narrowed.is_empty() { + return invalid("the selectors leave none of the job's nodes"); + } + Ok((transaction, narrowed)) +} diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs index c340fcc693..46c0249b48 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs @@ -1,5 +1,6 @@ //! Metadata reads hold live permission checks and results in the same transaction. +use crate::runs::types::RunsListing; use uuid::Uuid; use crate::authorization::main::authorize_listing::authorize_listing; @@ -13,7 +14,7 @@ use crate::store::models::{ControlStore, TransactionMode}; pub async fn list_person_runs( store: &ControlStore, (context, credentials): (&AuditContext, Uuid), - (location_id, target, limit): (Uuid, &str, u32), + (location_id, target, listing): (Uuid, &str, RunsListing), default_role: DefaultRole, ) -> Result, ControlError> { let (transaction, _) = authorize_listing( @@ -23,7 +24,7 @@ pub async fn list_person_runs( default_role, ) .await?; - let (transaction, runs) = list_runs_in(transaction, (location_id, target), limit).await?; + let (transaction, runs) = list_runs_in(transaction, (location_id, target), listing).await?; commit(transaction).await?; Ok(runs) } diff --git a/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs b/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs index b7bf47534c..8a056d0877 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/list_runs.rs @@ -4,6 +4,7 @@ use uuid::Uuid; use crate::runs::_helpers::lifecycle::reads::list_runs_in; use crate::runs::models::RunRecord; +use crate::runs::types::RunsListing; use crate::store::errors::ControlError; use crate::store::main::begin::begin; use crate::store::main::commit::commit; @@ -16,7 +17,12 @@ pub async fn list_runs( limit: u32, ) -> Result, ControlError> { let transaction = begin(store, TransactionMode::Read).await?; - let (transaction, runs) = list_runs_in(transaction, (location_id, target), limit).await?; + let (transaction, runs) = list_runs_in( + transaction, + (location_id, target), + RunsListing::Newest(limit, None), + ) + .await?; commit(transaction).await?; Ok(runs) } diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs index 1ce07b5c0f..35963b547e 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_job_run.rs @@ -3,18 +3,19 @@ use crate::credentials::models::AuditContext; use crate::runs::_helpers::requests::manual_job::request_job_run_in; use crate::runs::models::JobRunRequest; +use crate::runs::types::CommandPlanner; use crate::store::errors::ControlError; use crate::store::main::write::write; use crate::store::models::ControlStore; -/// The context is a trusted control-store owner, not a role supplied by an HTTP caller. +/// The context is a trusted control-store owner, not a role supplied by an HTTP caller; without a planner the job cannot be narrowed. pub async fn request_job_run( store: &ControlStore, - context: &AuditContext, + (context, planner): (&AuditContext, Option<&dyn CommandPlanner>), request: &JobRunRequest, ) -> Result { write(store, |transaction| { - request_job_run_in(transaction, (context, None), request) + request_job_run_in(transaction, (context, None), (planner, request)) }) .await } diff --git a/crates/sqlbuild-orchestration/src/runs/main/request_person_run.rs b/crates/sqlbuild-orchestration/src/runs/main/request_person_run.rs index c88868b7e0..6944230023 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/request_person_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/request_person_run.rs @@ -6,12 +6,13 @@ use crate::credentials::models::AuditContext; use crate::runs::_helpers::requests::adhoc::request_adhoc_run_in; use crate::runs::_helpers::requests::cancel::cancel_person_run; use crate::runs::_helpers::requests::manual_job::request_job_run_in; +use crate::runs::_helpers::requests::plan::request_plan_in; use crate::runs::models::PersonRunRequest; use crate::store::errors::ControlError; use crate::store::main::write::write; use crate::store::models::ControlStore; -/// A person queues a published job or an ad-hoc command, or cancels a run; each is authorised on the nodes it covers. +/// A person queues a published job, an ad-hoc command or a plan, or cancels a run; each is authorised on the nodes it covers. pub async fn request_person_run( store: &ControlStore, (context, credentials): (&AuditContext, Uuid), @@ -21,9 +22,25 @@ pub async fn request_person_run( PersonRunRequest::Cancel(request) => { cancel_person_run(store, (context, credentials), request).await } - PersonRunRequest::Job(request) => { + PersonRunRequest::Job(planner, request) => { write(store, |transaction| async move { - request_job_run_in(transaction, (context, Some(credentials)), request).await + request_job_run_in( + transaction, + (context, Some(credentials)), + (Some(planner), request), + ) + .await + }) + .await + } + PersonRunRequest::Plan(planner, (location_id, target, request)) => { + write(store, |transaction| async move { + request_plan_in( + transaction, + (context, credentials, planner), + (location_id, target, request), + ) + .await }) .await } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 50649157a9..12d84969d5 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -160,6 +160,9 @@ pub struct RunRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub window: Option, pub concurrency_groups: BTreeSet, + /// The selectors a person narrowed a job run with; reruns copy it with the rest of the request. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub narrowing: Option, } /// A trusted operator names a published job; execution inputs are never supplied by the caller. @@ -177,6 +180,12 @@ pub struct JobRunRequest { /// Rebuild every selected model from scratch; it rewrites history, as a window does. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub full_refresh: bool, + /// Narrow the job with `sqb build` selectors; only nodes already in the job can remain. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub select: Vec, + /// Leave nodes out of the job with `sqb build` selectors. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub exclude: Vec, } /// A retry derives its execution scope from a terminal run, never from caller selectors. @@ -188,6 +197,9 @@ pub struct RerunRequest { pub from_failure: bool, pub reason: String, pub priority: i64, + /// Rerun only these nodes of the source run; each must be in its selection. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub nodes: Vec, } /// Browser cancellation cannot supply an actor, claim or lifecycle transition. @@ -230,6 +242,65 @@ pub struct RunRecord { pub selection_json: BTreeSet, pub node_counts_json: BTreeMap, pub last_event_storage_id: i64, + /// How a person narrowed this run of a job, or the run it reruns, as typed; `None` when it runs the whole job. + #[serde( + default, + deserialize_with = "crate::runs::_helpers::lifecycle::narrowing::frozen_narrowing" + )] + pub narrowing: Option, + /// The cursor window a backfill processes; `None` for every other run. + #[serde( + default, + alias = "backfill_window", + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::runs::_helpers::lifecycle::narrowing::frozen_window" + )] + pub window: Option, + /// Set only when listing the queue: where the run stands and what it waits on. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue: Option, +} + +/// A queued run's place in claim order within its location and target, and what holds it back. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunQueueState { + /// 1-based, in the order the scheduler claims: priority, then queue time. + pub position: u32, + /// Empty when nothing holds the run back and it starts on a free worker. + pub waits: Vec, +} + +/// Why the scheduler cannot claim a queued run yet. Runs outside the listed location and target are counted, never named. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum QueueWait { + /// Its group run has not started yet, or is stopping. + GroupParent { run_id: Uuid }, + /// A job this one runs after in the same group has not finished. + UpstreamJob { run_id: Uuid, job_key: String }, + /// Another run of the same job is running. + JobRunning { run_id: Uuid }, + /// Another run holds write claims on nodes this run writes. + WriteClaim { + run_id: Option, + node_key: String, + other_nodes: u32, + }, + /// Running runs hold slots of a concurrency limit this run takes. + ConcurrencyLimit { + limit: String, + holders: Vec, + elsewhere: u32, + }, + /// An earlier queued run takes the same concurrency limit, so it goes first. + EarlierRun { run_id: Option, limit: String }, +} + +/// The `sqb build` selectors one run of a job was narrowed with. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct RunNarrowing { + pub select: Vec, + pub exclude: Vec, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] @@ -517,6 +588,22 @@ pub enum RunAuthority<'a> { Operator(&'a crate::credentials::models::AuditContext), } +/// A running run's hold on a write-claimed node or a concurrency slot, with where the run is. +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct QueueHolder { + pub(crate) held: String, + pub(crate) run_id: Uuid, + pub(crate) location_id: Uuid, + pub(crate) target: String, +} + +/// A run of a job that holds the job while it runs. +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct RunningJob { + pub(crate) run_id: Uuid, + pub(crate) job_key: String, +} + #[derive(Clone, Debug, Deserialize)] pub(crate) struct QueueCandidate { pub(crate) request_json: RunRequest, @@ -681,7 +768,12 @@ pub struct AdhocCheck { /// A person's change to the run queue: a published job, a command the planner resolves on the server, or a cancellation. pub enum PersonRunRequest<'a> { - Job(&'a JobRunRequest), + Job(&'a dyn CommandPlanner, &'a JobRunRequest), Adhoc(&'a dyn CommandPlanner, &'a AdhocRunRequest), Cancel(&'a CancelRunRequest), + /// A build command planned on the location for `(location_id, target)`; it never becomes a run. + Plan( + &'a dyn CommandPlanner, + (Uuid, &'a str, &'a crate::platform::models::PlanRequest), + ), } diff --git a/crates/sqlbuild-orchestration/src/runs/types.rs b/crates/sqlbuild-orchestration/src/runs/types.rs index fba6ad6c7f..603316a05d 100644 --- a/crates/sqlbuild-orchestration/src/runs/types.rs +++ b/crates/sqlbuild-orchestration/src/runs/types.rs @@ -1,17 +1,31 @@ //! The command planner the server is built with; the CLI owns the grammar and the selector resolver. +use std::collections::{BTreeMap, BTreeSet}; + use crate::runs::models::{CommandPlan, PlannerResource}; -/// Parse `command` with the CLI's grammar and resolve its selectors over `resources`; pure and side-effect free. +/// Each published job and job group's exact `kind:name` selection, by name. +pub type PublishedJobs = BTreeMap>; + +/// Parse `command` with the CLI's grammar and resolve its selectors over a publication; pure and side-effect free. pub trait CommandPlanner: Send + Sync { - /// The verb, exact node keys and validated settings, or why the command is refused. + /// The verb, exact node keys and validated settings, or why the command is refused; `job:` selects published jobs. fn plan( &self, command: &[String], - resources: &[PlannerResource], + publication: (&[PlannerResource], &PublishedJobs), ) -> Result; } +/// Which runs of one location and target to list. +#[derive(Clone, Copy, Debug)] +pub enum RunsListing { + /// Newest first: at most this many, after the queue time and ID of a previous page's last run. + Newest(u32, Option<(chrono::DateTime, uuid::Uuid)>), + /// Queued runs in claim order, at most this many, each with its queue state. + Queue(u32), +} + /// What the API hands runs to workers with: their Docker images and the command planner. pub type RunHandoff = ( Vec, diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index 9576f5d4a0..29ee7f287e 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -67,6 +67,7 @@ pub(crate) async fn persist_ticks( settings: trigger.job_json.settings.clone(), window: None, concurrency_groups: trigger.job_json.concurrency_groups.clone(), + narrowing: None, }; transaction = queue_run(transaction, &request).await?; } diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs index f1a299164c..af58285254 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs @@ -35,6 +35,8 @@ fn calendar(seconds: impl Iterator) -> TriggerDefinition { job_key: "orders".into(), starts_at: Some(time(0)), default_status: TriggerStatus::Running, + targets: BTreeSet::new(), + declared_at: None, schedule: Schedule::Calendar { times: seconds.map(time).collect(), refresh_at: time(2000), @@ -203,6 +205,8 @@ pub(super) fn simulate_cron(expression: &str, wall_times: &[(u32, u32)]) -> Resu }, starts_at: Some(start), default_status: TriggerStatus::default(), + targets: BTreeSet::new(), + declared_at: None, }; let strategy = proptest::collection::vec((-1_440_i64..5_760, any::()), 0..256); let mut property_runner = TestRunner::new(Config { diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs index 56f0b424fe..0e3f093e53 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs @@ -1,15 +1,33 @@ //! Sensor evaluation phases are recorded once; the latest cursor lives on the trigger's state. +use std::collections::BTreeSet; + use serde_json::json; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::sensors::models::{ - SensorCursor, SensorEvaluationFact, SensorEvaluationOutcome, SensorEvaluationRequest, + SensorCursor, SensorDefinition, SensorEvaluationFact, SensorEvaluationOutcome, + SensorEvaluationRequest, }; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{Change, ControlTransaction, Parameter}; +/// Evaluations record the fingerprint of the definition they ran, so a later definition shows as changed; where it is written is not part of it. +pub(crate) fn definition_fingerprint( + definition: &SensorDefinition, +) -> Result { + let behaviour: SensorDefinition = SensorDefinition { + targets: BTreeSet::new(), + declared_at: None, + ..definition.clone() + }; + Ok(sha256_hex(&serde_json::to_vec(&behaviour).map_err( + |error| ControlError::Invalid(error.to_string()), + )?)) +} + pub(crate) async fn append_evaluation( transaction: ControlTransaction, request: &SensorEvaluationRequest, diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs index 69a24fad30..5c7c6de009 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs @@ -48,8 +48,11 @@ pub(crate) async fn queue_sensor_run( priority: 0, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }; - let (transaction, mut frozen) = frozen_job_request(transaction, &context, &authored).await?; + let (transaction, mut frozen) = + frozen_job_request(transaction, (&context, None), &authored).await?; if frozen.snapshot_id != evaluation.snapshot_id { return Err(ControlError::Fenced); } diff --git a/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs index 30c6503463..8086ed0ca7 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/begin_sensor_evaluation.rs @@ -1,7 +1,6 @@ //! Commit the requested phase before evaluation, so crashes count as attempts. use chrono::TimeDelta; -use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use uuid::Uuid; use crate::declarations::models::TriggerStatus; @@ -9,7 +8,7 @@ use crate::scheduler::main::check_lease::check_lease; use crate::scheduler::models::SchedulerLease; use crate::sensors::_helpers::attempts::{defer_sensor, last_attempt, read_sensor}; use crate::sensors::_helpers::custom::request_evaluation_work; -use crate::sensors::_helpers::facts::append_evaluation; +use crate::sensors::_helpers::facts::{append_evaluation, definition_fingerprint}; use crate::sensors::main::read_sensor_cursor::read_sensor_cursor; use crate::sensors::models::{SensorEvaluationOutcome, SensorEvaluationRequest}; use crate::store::errors::ControlError; @@ -58,10 +57,7 @@ pub async fn begin_sensor_evaluation( return Ok(None); } } - let definition_fingerprint = sha256_hex( - &serde_json::to_vec(&sensor.definition_json) - .map_err(|error| ControlError::Invalid(error.to_string()))?, - ); + let definition_fingerprint = definition_fingerprint(&sensor.definition_json)?; let request = SensorEvaluationRequest { evaluation_id: Uuid::now_v7(), trigger_id: sensor.trigger_id, diff --git a/crates/sqlbuild-orchestration/src/sensors/main/mod.rs b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs index e819eb77e1..e114a95484 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/mod.rs @@ -2,6 +2,7 @@ pub mod begin_sensor_evaluation; pub mod complete_builtin_sensor; pub mod complete_custom_sensor; pub(crate) mod publish_sensors; +pub mod read_person_sensor; pub(crate) mod read_sensor_cursor; pub(crate) mod reset_sensor_cursor; pub(crate) mod validate_sensors; diff --git a/crates/sqlbuild-orchestration/src/sensors/main/read_person_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/read_person_sensor.rs new file mode 100644 index 0000000000..5a29f75caa --- /dev/null +++ b/crates/sqlbuild-orchestration/src/sensors/main/read_person_sensor.rs @@ -0,0 +1,155 @@ +//! A person sees a sensor's cursor, recent checks and run requests only with view on every current node; values the sensor computed also need view.evidence. + +use std::collections::BTreeMap; + +use sqlbuild_core::redaction::main::redact_text::redact_text; +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::authorization::main::person_holds_target::person_holds_target; +use crate::credentials::models::{Action, AuditContext, DefaultRole}; +use crate::sensors::_helpers::facts::definition_fingerprint; +use crate::sensors::models::{ + PersonSensorCursor, PersonSensorEvaluation, PersonSensorRunRequest, PersonSensorState, + SensorCursor, SensorEvaluationOutcome, StoredEvaluationEvent, StoredRunRequest, StoredSensor, +}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query_records::query_records}; +use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; + +/// `None` when no sensor by that ID is in this location and target; authorization runs first either way. +pub async fn read_person_sensor( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + ((location_id, target), (trigger_id, limit)): ((Uuid, &str), (Uuid, u32)), + default_role: DefaultRole, +) -> Result, ControlError> { + let evidence = person_holds_target( + store, + credentials, + (Action::ViewEvidence, location_id, target), + default_role, + ) + .await?; + let (transaction, _) = authorize_listing( + begin(store, TransactionMode::Write).await?, + (context, credentials), + (location_id, target), + default_role, + ) + .await?; + let (transaction, sensors): (_, Vec) = query_records( + transaction, + "SELECT t.definition_json, s.cursor_json FROM sqb_triggers t \ + LEFT JOIN sqb_trigger_state s ON s.trigger_id=t.trigger_id \ + WHERE t.trigger_id=$1 AND t.location_id=$2 AND t.target=$3 AND t.kind='sensor'", + vec![ + Parameter::Id(trigger_id), + Parameter::Id(location_id), + Parameter::Text(target.into()), + ], + ) + .await?; + let Some(sensor) = sensors.into_iter().next() else { + commit(transaction).await?; + return Ok(None); + }; + let limit = Parameter::Integer(i64::from(limit.min(1000))); + let (transaction, evaluations) = + evaluations(transaction, (trigger_id, limit.clone()), evidence).await?; + let (transaction, requests): (_, Vec) = query_records( + transaction, + "SELECT q.run_key, q.run_id, q.evaluation_id, r.queued_at, r.status FROM sqb_run_requests q \ + JOIN sqb_runs r ON r.run_id=q.run_id WHERE q.trigger_id=$1 \ + ORDER BY r.queued_at DESC, q.run_key LIMIT $2", + vec![Parameter::Id(trigger_id), limit], + ) + .await?; + commit(transaction).await?; + Ok(Some(PersonSensorState { + cursor: sensor + .cursor_json + .as_ref() + .map(|cursor| shown_cursor(cursor, evidence)), + definition_fingerprint: definition_fingerprint(&sensor.definition_json)?, + evaluations, + run_requests: requests + .into_iter() + .map(|request| PersonSensorRunRequest { + run_key: if evidence { + redact_text(&request.run_key) + } else { + String::new() + }, + run_id: request.run_id, + evaluation_id: request.evaluation_id, + queued_at: request.queued_at, + status: request.status, + }) + .collect(), + })) +} + +/// The newest `limit` checks, each projected from its recorded phases. +async fn evaluations( + transaction: ControlTransaction, + (trigger_id, limit): (Uuid, Parameter), + evidence: bool, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, events): (_, Vec) = query_records( + transaction, + "SELECT occurred_at, fact_json FROM sqb_sensor_evaluation_events WHERE trigger_id=$1 \ + AND evaluation_id IN (SELECT evaluation_id FROM sqb_sensor_evaluation_events \ + WHERE trigger_id=$1 AND phase='requested' ORDER BY occurred_at DESC LIMIT $2)", + vec![Parameter::Id(trigger_id), limit], + ) + .await?; + let mut checks: BTreeMap = BTreeMap::new(); + for event in events { + let request = &event.fact_json.request; + let check = checks + .entry(request.evaluation_id) + .or_insert_with(|| PersonSensorEvaluation { + evaluation_id: request.evaluation_id, + phase: SensorEvaluationOutcome::Requested.phase().into(), + requested_at: request.requested_at, + finished_at: None, + cursor_before: shown_cursor(&request.cursor, evidence), + cursor_after: None, + run_count: 0, + failure_code: None, + definition_fingerprint: request.definition_fingerprint.clone(), + }); + let outcome = event.fact_json.outcome; + if outcome == SensorEvaluationOutcome::Requested { + continue; + } + check.phase = outcome.phase().into(); + check.finished_at = Some(event.occurred_at); + match outcome { + SensorEvaluationOutcome::Completed { cursor, run_ids } => { + check.cursor_after = Some(shown_cursor(&cursor, evidence)); + check.run_count = run_ids.len(); + } + SensorEvaluationOutcome::Failed { code } => check.failure_code = Some(code), + _ => {} + } + } + let mut checks: Vec = checks.into_values().collect(); + checks.sort_by(|left, right| { + (right.requested_at, right.evaluation_id).cmp(&(left.requested_at, left.evaluation_id)) + }); + Ok((transaction, checks)) +} + +/// A custom value is what the sensor computed from data it read, so it shows only with view.evidence. +fn shown_cursor(cursor: &SensorCursor, evidence: bool) -> PersonSensorCursor { + PersonSensorCursor { + event_storage_id: cursor.event_storage_id, + custom: cursor + .custom + .as_deref() + .filter(|_| evidence) + .map(redact_text), + } +} diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index e0cfc25b6c..33e8c7c098 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::declarations::models::TriggerStatus; +use crate::declarations::models::{DeclaredAt, TriggerStatus}; use crate::platform::models::PlatformEventKind; use crate::runs::models::{NodeOutcome, RunOutcome}; @@ -104,6 +104,11 @@ pub struct SensorDefinition { pub minimum_interval_seconds: NonZeroU32, #[serde(default)] pub default_status: TriggerStatus, + /// The targets the declaration is limited to; empty for every target. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub targets: BTreeSet, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub declared_at: Option, } /// Storage order is authoritative even when a fact's occurrence time predates earlier pages. @@ -190,3 +195,73 @@ pub(crate) struct DueSensor { pub(crate) definition_json: SensorDefinition, pub(crate) status: TriggerStatus, } + +/// A sensor's committed position and recent checks, for a person who may list the target. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct PersonSensorState { + /// Absent until the sensor first commits a cursor. + pub cursor: Option, + /// Fingerprint of the published definition; a check with another one ran older code. + pub definition_fingerprint: String, + /// Newest first. + pub evaluations: Vec, + /// Runs the sensor requested, by run key, newest first. + pub run_requests: Vec, +} + +/// The control event position built-in conditions read from, and a custom sensor's own value after secret redaction; the value needs view.evidence. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct PersonSensorCursor { + pub event_storage_id: i64, + pub custom: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct PersonSensorEvaluation { + pub evaluation_id: Uuid, + /// The latest phase: `requested` alone means the check is in flight; otherwise `completed`, `failed`, `timed_out` or `abandoned`. + pub phase: String, + pub requested_at: DateTime, + pub finished_at: Option>, + pub cursor_before: PersonSensorCursor, + /// Present when the check completed. + pub cursor_after: Option, + pub run_count: usize, + /// The failure code of a failed check. + pub failure_code: Option, + pub definition_fingerprint: String, +} + +/// A run key is redacted and withheld like the custom cursor it usually derives from: empty without view.evidence. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct PersonSensorRunRequest { + pub run_key: String, + pub run_id: Uuid, + pub evaluation_id: Uuid, + pub queued_at: DateTime, + pub status: crate::runs::models::RunStatus, +} + +/// A published sensor and its committed cursor. +#[derive(Debug, Deserialize)] +pub(crate) struct StoredSensor { + pub(crate) definition_json: SensorDefinition, + pub(crate) cursor_json: Option, +} + +/// One recorded phase of a sensor check. +#[derive(Debug, Deserialize)] +pub(crate) struct StoredEvaluationEvent { + pub(crate) occurred_at: DateTime, + pub(crate) fact_json: SensorEvaluationFact, +} + +/// A run key the sensor requested and the run it made. +#[derive(Debug, Deserialize)] +pub(crate) struct StoredRunRequest { + pub(crate) run_key: String, + pub(crate) run_id: Uuid, + pub(crate) evaluation_id: Uuid, + pub(crate) queued_at: DateTime, + pub(crate) status: crate::runs::models::RunStatus, +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/holds_on_current_nodes.rs b/crates/sqlbuild-orchestration/src/snapshots/main/holds_on_current_nodes.rs new file mode 100644 index 0000000000..0110ab2067 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/holds_on_current_nodes.rs @@ -0,0 +1,35 @@ +//! Decide whether a person holds one action on every current node of a target, for checks that must not audit. + +use std::collections::BTreeSet; + +use uuid::Uuid; + +use crate::authorization::main::decide_person_scopes::decide_person_scopes; +use crate::credentials::models::{Action, PermissionScope}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Whether the person holds `action` on every node of the target's current snapshot, decided without recording a denial. +pub(crate) async fn holds_on_current_nodes( + transaction: ControlTransaction, + principal_id: Uuid, + (action, location_id, target): (Action, Uuid, &str), +) -> Result<(ControlTransaction, bool), ControlError> { + let (transaction, current) = query(transaction, + "SELECT n.node_key FROM sqb_snapshot_nodes n JOIN sqb_current_snapshots c ON c.snapshot_id=n.snapshot_id \ + WHERE c.location_id=$1 AND c.target=$2", + vec![Parameter::Id(location_id), Parameter::Text(target.into())]).await?; + let scope = PermissionScope { + action, + location_id: Some(location_id), + target: Some(target.into()), + nodes: current + .iter() + .filter_map(|row| row["node_key"].as_str().map(str::to_owned)) + .collect::>(), + }; + let (transaction, refusals) = + decide_person_scopes(transaction, principal_id, (target, &[scope])).await?; + Ok((transaction, refusals.as_deref() == Some(&[None]))) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index cfb2475cc5..c06e3c034a 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -1,6 +1,7 @@ pub(super) mod _publish_snapshot; pub mod current_snapshot; pub mod current_snapshot_record; +pub(crate) mod holds_on_current_nodes; pub(crate) mod job_write_set; pub mod list_person_snapshots; pub mod next_remote_generation; diff --git a/crates/sqlbuild-orchestration/src/snapshots/models.rs b/crates/sqlbuild-orchestration/src/snapshots/models.rs index 82c6f9d5a3..77c3558329 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/models.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/models.rs @@ -113,6 +113,10 @@ impl SnapshotRetention { pub(crate) fn kept(self) -> i64 { i64::from(self.count) } + + pub(crate) fn kept_count(self) -> u16 { + self.count + } } impl Default for SnapshotRetention { diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index e71dcc67ad..f0c0c2297e 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -600,6 +600,10 @@ impl StatisticsRetention { pub(crate) fn cutoff(self, now: DateTime) -> DateTime { now - Duration::days(i64::from(self.days)) } + + pub(crate) fn kept_days(self) -> u16 { + self.days + } } impl Default for StatisticsRetention { diff --git a/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs b/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs new file mode 100644 index 0000000000..adc0e0fa7c --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs @@ -0,0 +1,112 @@ +//! People, requests and counts the plan contract builds on. + +use uuid::Uuid; + +use crate::authorization::main::assign_builtin_role::assign_builtin_role; +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::main::sessions::set_password::set_password; +use crate::credentials::models::{ + AuditContext, BuiltinRole, PasswordAuthority, RoleAssignmentRequest, +}; +use crate::platform::main::report_location_health::report_location_health; +use crate::platform::models::{ + LocationHealth, LocationHealthReport, LocationWorkPoll, PlanRequest, +}; +use crate::runs::main::request_person_run::request_person_run; +use crate::runs::models::PersonRunRequest; +use crate::runs::tests::test_types::ExactKeyPlanner; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{ControlStore, TransactionMode}; + +pub(crate) fn plan_request(tokens: &[&str]) -> PlanRequest { + PlanRequest { + request_id: Uuid::now_v7(), + command: tokens.iter().map(|token| (*token).to_owned()).collect(), + } +} + +/// Request a plan on the `development` target as `person`. +pub(crate) async fn ask( + store: &ControlStore, + (context, person): (&AuditContext, Uuid), + location: Uuid, + request: &PlanRequest, +) -> Result { + request_person_run( + store, + (context, person), + PersonRunRequest::Plan(&ExactKeyPlanner, (location, "development", request)), + ) + .await +} + +/// A principal with a password and, when given, a built-in role on the location's `development` target. +pub(crate) async fn person( + store: &ControlStore, + (context, admin): (&AuditContext, Uuid), + (name, location): (&str, Uuid), + roles: &[BuiltinRole], +) -> Uuid { + let person = create_principal(store, context, name) + .await + .expect("person"); + set_password( + store, + context, + PasswordAuthority::Operator(person), + "orders-password-2026", + ) + .await + .expect("password"); + for role in roles { + assign_builtin_role( + store, + (context, admin), + &RoleAssignmentRequest { + principal_id: person, + role: (*role).into(), + location_id: Some(location), + target: Some("development".into()), + revoke: false, + }, + ) + .await + .expect("assign"); + } + person +} + +pub(crate) async fn register_instance(store: &ControlStore, location: Uuid) -> LocationWorkPoll { + let poll = LocationWorkPoll { + session_id: Uuid::now_v7(), + target: "development".into(), + }; + report_location_health( + store, + location, + &LocationHealthReport { + event_id: Uuid::now_v7(), + session_id: poll.session_id, + target: poll.target.clone(), + health: LocationHealth::Heartbeat {}, + }, + ) + .await + .expect("register instance"); + poll +} + +/// Run and run-node rows together; a plan must add none. +pub(crate) async fn run_rows(store: &ControlStore) -> i64 { + let transaction = begin(store, TransactionMode::Read).await.expect("begin"); + let (transaction, rows) = query( + transaction, + "SELECT (SELECT COUNT(*) FROM sqb_runs) + (SELECT COUNT(*) FROM sqb_run_nodes) AS n", + Vec::new(), + ) + .await + .expect("count"); + commit(transaction).await.expect("commit"); + rows[0]["n"].as_i64().expect("count") +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_changes.rs b/crates/sqlbuild-orchestration/tests/api/test_changes.rs index 4db85856e1..5262110726 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_changes.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_changes.rs @@ -177,9 +177,11 @@ fn given_independent_http_observers_when_committed_state_changes_then_scoped_sse priority: 0, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }; assert!( - request_job_run(store, &audit, &request) + request_job_run(store, (&audit, None), &request) .await .expect("queue job") ); @@ -191,7 +193,7 @@ fn given_independent_http_observers_when_committed_state_changes_then_scoped_sse expect_topics(&mut a, &run_topics).await; expect_topics(&mut b, &run_topics).await; assert!( - !request_job_run(store, &audit, &request) + !request_job_run(store, (&audit, None), &request) .await .expect("idempotent retry") ); diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index a1b7ab1c0d..025a9bb80b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -22,6 +22,8 @@ mod output_helpers; mod person_sessions_helpers; #[path = "person_sessions/test_types.rs"] mod person_sessions_test_types; +#[path = "plans/helpers.rs"] +mod plans_helpers; #[path = "recall/helpers.rs"] mod recall_helpers; #[path = "statistics/eligibility/helpers.rs"] @@ -68,6 +70,8 @@ mod test_passwords; mod test_person_live; #[path = "test_person_location_work.rs"] mod test_person_location_work; +#[path = "test_person_plans.rs"] +mod test_person_plans; #[path = "test_person_sessions.rs"] mod test_person_sessions; #[path = "test_proxy.rs"] @@ -80,6 +84,8 @@ mod test_reads; mod test_server_errors; #[path = "test_statistics_recovery.rs"] mod test_statistics_recovery; +#[path = "test_trigger_upcoming.rs"] +mod test_trigger_upcoming; #[path = "test_types.rs"] mod test_types; #[path = "test_ui_assets.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_denials.rs b/crates/sqlbuild-orchestration/tests/api/test_denials.rs index 3983b08d29..5aa903401b 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_denials.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_denials.rs @@ -92,9 +92,11 @@ fn given_refused_requests_when_people_send_them_then_no_node_is_named_and_unknow priority: 0, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }; assert!( - request_job_run(store, &roles.context, &queued) + request_job_run(store, (&roles.context, None), &queued) .await .expect("queued run") ); diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_live.rs b/crates/sqlbuild-orchestration/tests/api/test_person_live.rs index 31b6c4250a..09f414f988 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_live.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_live.rs @@ -170,9 +170,11 @@ fn given_people_with_different_roles_when_they_bootstrap_and_subscribe_then_only priority: 0, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }; assert!( - request_job_run(store, &roles.context, &request) + request_job_run(store, (&roles.context, None), &request) .await .expect("queue job") ); diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs new file mode 100644 index 0000000000..424004c9ef --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs @@ -0,0 +1,376 @@ +//! A person's plan is frozen like an ad-hoc build, queued as location work and read only by them. + +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::plans_helpers::{ask, person, plan_request, register_instance, run_rows}; +use crate::api::tests::test_types::PlanTestCase; +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::models::{AuditContext, BuiltinRole}; +use crate::credentials::tests::location_helpers::create_location; +use crate::platform::main::claim_location_work::claim_location_work; +use crate::platform::main::complete_location_work::complete_location_work; +use crate::platform::main::read_person_plan::read_person_plan; +use crate::platform::main::request_location_work::request_location_work; +use crate::platform::models::{ + LocationWork, LocationWorkOutcome, LocationWorkRequest, LocationWorkResult, PlanRequest, +}; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_work_for_their_requester() + { + let test_cases = [ + PlanTestCase { + description: "SQLite plans", + backend: Backend::Sqlite, + expected_selection: &["customers", "orders"], + expected_warning: "connect failed: db_password=[REDACTED]", + }, + PlanTestCase { + description: "PostgreSQL plans", + backend: Backend::Postgres, + expected_selection: &["customers", "orders"], + expected_warning: "connect failed: db_password=[REDACTED]", + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: Some(std::net::Ipv4Addr::LOCALHOST.into()), + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let location = create_location(store, "orders", "test-owner") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish"); + let owner = (&context, admin); + let viewer = person( + store, + owner, + ("orders-viewer", location), + &[BuiltinRole::Viewer], + ) + .await; + let developer = person( + store, + owner, + ("orders-developer", location), + &[BuiltinRole::Developer], + ) + .await; + let analyst = person( + store, + owner, + ("orders-analyst", location), + &[BuiltinRole::Viewer], + ) + .await; + let auditor = person( + store, + owner, + ("orders-auditor", location), + &[BuiltinRole::Viewer], + ) + .await; + let outsider = person(store, owner, ("orders-outsider", location), &[]).await; + let first = plan_request(&[ + "build", + "customers", + "orders", + "--full-refresh", + "--warehouse", + "WH_PLAN", + ]); + + let silent = ask(store, (&context, viewer), location, &first).await; + assert!( + matches!(&silent, Err(ControlError::Invalid(message)) if message.contains("sqb location serve")), + "{description}: no live location, got {silent:?}", + description = test_case.description + ); + let poll = register_instance(store, location).await; + assert!( + matches!( + ask(store, (&context, outsider), location, &first).await, + Err(ControlError::Denied { .. }) + ), + "{description}: view is required", + description = test_case.description + ); + assert!( + matches!( + ask(store, (&context, viewer), location, &plan_request(&["seed", "customers"])).await, + Err(ControlError::Invalid(message)) if message.contains("cannot be planned") + ), + "{description}: only builds are planned", + description = test_case.description + ); + assert!( + ask(store, (&context, viewer), location, &first) + .await + .expect("viewer plans"), + "{description}: a viewer may plan", + description = test_case.description + ); + assert!( + !ask(store, (&context, viewer), location, &first) + .await + .expect("idempotent retry"), + "{description}: a retry queues nothing", + description = test_case.description + ); + let altered = PlanRequest { + command: vec!["build".into(), "inventory".into()], + ..first.clone() + }; + assert!( + ask(store, (&context, viewer), location, &altered) + .await + .is_err(), + "{description}: a request ID cannot carry another command", + description = test_case.description + ); + assert!( + ask(store, (&context, developer), location, &first) + .await + .is_err(), + "{description}: a request ID cannot carry another actor", + description = test_case.description + ); + assert!( + matches!( + ask(store, (&context, viewer), location, &plan_request(&["build", "inventory"])).await, + Err(ControlError::Invalid(message)) if message.contains("still running") + ), + "{description}: one unfinished plan per person", + description = test_case.description + ); + for queued in [developer, analyst, auditor] { + assert!( + ask( + store, + (&context, queued), + location, + &plan_request(&["build", "products"]) + ) + .await + .expect("queue has room"), + "{description}: queue has room", + description = test_case.description + ); + } + assert!( + matches!( + ask(store, (&context, admin), location, &plan_request(&["build", "products"])).await, + Err(ControlError::Invalid(message)) if message.contains("already waiting") + ), + "{description}: four plans per target", + description = test_case.description + ); + let direct = LocationWorkRequest { + request_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + reason: "plan".into(), + work: LocationWork::Plan { + command: vec!["build".into(), "orders".into()], + selection: ["orders".to_owned()].into(), + settings: Default::default(), + }, + }; + assert!( + request_location_work(store, (&context, None), &direct) + .await + .is_err(), + "{description}: plans only come from a person's frozen command", + description = test_case.description + ); + let connection = LocationWorkRequest { + request_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + reason: "Check orders connectivity".into(), + work: LocationWork::CheckConnection {}, + }; + request_location_work(store, (&context, None), &connection) + .await + .expect("connection test"); + let claimed = claim_location_work(store, location, &poll) + .await + .expect("claim") + .expect("work"); + assert_eq!( + claimed.request.request_id, + connection.request_id, + "{description}: other location work goes before plans", + description = test_case.description + ); + complete_location_work( + store, + (location, connection.request_id), + &LocationWorkResult { + claim_id: claimed.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Connection { healthy: true }, + }, + ) + .await + .expect("complete connection"); + let claimed = claim_location_work(store, location, &poll) + .await + .expect("claim") + .expect("plan"); + assert_eq!( + claimed.request.request_id, + first.request_id, + "{description}", + description = test_case.description + ); + let LocationWork::Plan { + selection, + settings, + .. + } = &claimed.request.work + else { + panic!( + "{description}: plan work", + description = test_case.description + ) + }; + assert_eq!( + selection.iter().map(String::as_str).collect::>(), + test_case.expected_selection, + "{description}", + description = test_case.description + ); + assert_eq!( + settings["full_refresh"], + json!(true), + "{description}", + description = test_case.description + ); + assert!( + !settings.contains_key("warehouse"), + "{description}: a plan never takes a named warehouse", + description = test_case.description + ); + let lease = (claimed.lease_expires_at - chrono::Utc::now()).num_seconds(); + assert!( + (240..=300).contains(&lease), + "{description}: plan lease {lease}", + description = test_case.description + ); + assert_eq!( + run_rows(store).await, + 0, + "{description}: a plan is never a run", + description = test_case.description + ); + + let complete = |document: serde_json::Value| LocationWorkResult { + claim_id: claimed.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Plan { document }, + }; + for invalid in [ + json!(["models"]), + json!({"selected_count": 1}), + json!({"models": "x".repeat(9 * 1024 * 1024)}), + ] { + assert!( + complete_location_work(store, (location, first.request_id), &complete(invalid)) + .await + .is_err(), + "{description}: shape and size are checked", + description = test_case.description + ); + } + let document = json!({ + "models": [{"name": "orders", "action": "create_table", "reason": "first_run"}], + "warnings": [{"severity": "warning", "message": "connect failed: db_password=orders-pass"}], + "python_nodes": [{"name": "orders_export", "identity_diff": {"source_diff": ["-a", "+b"]}}], + }); + assert!( + complete_location_work( + store, + (location, first.request_id), + &complete(document.clone()) + ) + .await + .expect("complete plan"), + "{description}", + description = test_case.description + ); + assert!( + !complete_location_work(store, (location, first.request_id), &complete(document)) + .await + .expect("same result retried after redaction"), + "{description}", + description = test_case.description + ); + + let read = |reader: Uuid| { + read_person_plan( + store, + (&context, reader), + (location, "development", first.request_id), + ) + }; + let record = read(viewer).await.expect("read").expect("own plan"); + let Some(LocationWorkOutcome::Plan { document }) = + record.result_json.map(|result| result.outcome) + else { + panic!( + "{description}: plan result", + description = test_case.description + ) + }; + assert_eq!( + document["warnings"][0]["message"], + json!(test_case.expected_warning), + "{description}", + description = test_case.description + ); + assert_eq!( + document["models"][0]["reason"], + json!("first_run"), + "{description}", + description = test_case.description + ); + assert!( + document["python_nodes"][0].get("identity_diff").is_none(), + "{description}: viewers lack view.sql", + description = test_case.description + ); + assert!( + read(developer).await.expect("read").is_none(), + "{description}: only the requester reads a plan", + description = test_case.description + ); + assert!( + matches!(read(outsider).await, Err(ControlError::Denied { .. })), + "{description}: view is required to read", + description = test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs index d94cd61b45..f5558bc4cb 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -335,6 +335,7 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ "/api/admin/people".to_owned(), "/api/admin/roles".to_owned(), format!("/api/admin/locations/{location}/targets/development/roles"), + "/api/admin/server".to_owned(), ] { let listed = request( &provider.client, @@ -409,9 +410,11 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ jobs.as_array().into_iter().flatten().cloned().collect(); let expected_jobs = [ json!({"job_key": "nightly", "kind": "group", "description": "Nightly orders", - "selection": ["orders"], "concurrency_groups": [], "snapshot_id": published.snapshot_id}), + "selection": ["orders"], "concurrency_groups": [], "snapshot_id": published.snapshot_id, + "select": null, "declared_at": null}), json!({"job_key": "orders", "kind": "job", "description": "Refresh orders", - "selection": ["orders"], "concurrency_groups": ["warehouse_heavy"], "snapshot_id": published.snapshot_id}), + "selection": ["orders"], "concurrency_groups": ["warehouse_heavy"], "snapshot_id": published.snapshot_id, + "select": null, "declared_at": null}), ]; assert_eq!( listed_jobs, diff --git a/crates/sqlbuild-orchestration/tests/api/test_trigger_upcoming.rs b/crates/sqlbuild-orchestration/tests/api/test_trigger_upcoming.rs new file mode 100644 index 0000000000..5897709eb4 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_trigger_upcoming.rs @@ -0,0 +1,118 @@ +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use serde_json::json; +use uuid::Uuid; + +use crate::api::tests::test_types::UpcomingTestCase; +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::credentials::tests::location_helpers::create_location; +use crate::declarations::main::list_person_triggers::list_person_triggers; +use crate::scheduler::tests::contracts::ticks::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_running_and_paused_schedules_when_people_list_triggers_then_running_ones_carry_their_next_due_times() + { + let hourly = [ + "2031-01-01T01:00:00Z", + "2031-01-01T02:00:00Z", + "2031-01-01T03:00:00Z", + "2031-01-01T04:00:00Z", + "2031-01-01T05:00:00Z", + ]; + let calendar = [ + "2031-02-01T00:00:00Z", + "2031-03-01T00:00:00Z", + "2031-04-01T00:00:00Z", + ]; + let test_cases = [ + UpcomingTestCase { + description: "SQLite", + backend: Backend::Sqlite, + expected_hourly: hourly, + expected_calendar: calendar, + }, + UpcomingTestCase { + description: "PostgreSQL", + backend: Backend::Postgres, + expected_hourly: hourly, + expected_calendar: calendar, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.insert( + "triggers".into(), + json!({ + "hourly_orders": {"description": "Hourly orders", "job_key": "orders", + "starts_at": "2031-01-01T00:30:00Z", + "schedule": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"}}, + "paused_orders": {"description": "Paused orders", "job_key": "orders", + "default_status": "paused", "starts_at": "2031-01-01T00:30:00Z", + "schedule": {"kind": "cron", "expression": "0 * * * *", "timezone": "UTC"}}, + "month_start_orders": {"description": "Month start orders", "job_key": "orders", + "schedule": {"kind": "calendar", "times": test_case.expected_calendar, + "refresh_at": "2031-01-15T00:00:00Z"}} + }), + ); + publish_snapshot(store, &input).await.expect("publication"); + let audit = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let administrator = bootstrap_user(store, &audit, "orders-admin") + .await + .expect("administrator"); + let listed: BTreeMap>> = list_person_triggers( + store, + (&audit, administrator), + (location, "development"), + DefaultRole::None, + ) + .await + .expect("person triggers") + .into_iter() + .map(|record| (record.trigger.name, record.upcoming)) + .collect(); + let times = |texts: &[&str]| -> Vec> { + texts + .iter() + .map(|text| text.parse().expect("time")) + .collect() + }; + assert_eq!( + listed, + BTreeMap::from([ + ( + "hourly_orders".to_owned(), + times(&test_case.expected_hourly) + ), + ( + "month_start_orders".to_owned(), + times(&test_case.expected_calendar) + ), + ("paused_orders".to_owned(), Vec::new()), + ]), + "{}: five cron times from the start, the calendar's remaining dates, none while paused", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 0b878cb812..f5e8e9df1a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -213,3 +213,17 @@ pub(crate) struct OlderNodeFinished { error_code: Option, skipped_because_of: Option, } + +pub(crate) struct PlanTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_selection: &'static [&'static str], + pub(crate) expected_warning: &'static str, +} + +pub(crate) struct UpcomingTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_hourly: [&'static str; 5], + pub(crate) expected_calendar: [&'static str; 3], +} diff --git a/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs b/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs index e05086e73d..fa5ba26b5e 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/roles/helpers.rs @@ -23,6 +23,7 @@ use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::{PersonTriggerChange, TriggerChange, TriggerChangeRequest}; use crate::runs::main::list_person_runs::list_person_runs; use crate::runs::tests::helpers::snapshot; +use crate::runs::types::RunsListing; use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; @@ -323,10 +324,15 @@ pub(crate) async fn member_listings( .await .map(|_| ()) .map_err(denial), - list_person_runs(store, member, (scope.0, scope.1, 10), DefaultRole::None) - .await - .map(|_| ()) - .map_err(denial), + list_person_runs( + store, + member, + (scope.0, scope.1, RunsListing::Newest(10, None)), + DefaultRole::None, + ) + .await + .map(|_| ()) + .map_err(denial), ] } diff --git a/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs b/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs index 502b771942..db6d2c6c57 100644 --- a/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs +++ b/crates/sqlbuild-orchestration/tests/credentials/test_audit_retention.rs @@ -80,9 +80,11 @@ fn given_expired_audit_rows_when_pruned_then_undelivered_rows_survive_and_retrie priority: 1, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }; assert!( - request_job_run(store, &context, &request) + request_job_run(store, (&context, None), &request) .await .expect("request") ); @@ -136,7 +138,7 @@ fn given_expired_audit_rows_when_pruned_then_undelivered_rows_survive_and_retrie assert_eq!(ids, [undelivered, newest]); assert_eq!(deliveries, 0, "deliveries of pruned rows go with them"); assert!( - !request_job_run(store, &context, &request) + !request_job_run(store, (&context, None), &request) .await .expect("retry still finds its original request after the audit is pruned") ); diff --git a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs index eb9b98bffe..894a2240a1 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_contracts.rs @@ -2,6 +2,8 @@ #[path = "helpers.rs"] mod helpers; +#[path = "test_installation.rs"] +mod test_installation; #[path = "test_problems.rs"] mod test_problems; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/platform/test_installation.rs b/crates/sqlbuild-orchestration/tests/platform/test_installation.rs new file mode 100644 index 0000000000..63ac1fed38 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/platform/test_installation.rs @@ -0,0 +1,72 @@ +use crate::authorization::main::bootstrap_user::bootstrap_user; +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::models::AuditContext; +use crate::platform::main::read_installation::read_installation; +use crate::platform::models::StorageKind; +use crate::platform::tests::test_types::InstallationTestCase; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use uuid::Uuid; + +#[test] +fn given_a_migrated_store_when_people_read_the_installation_then_only_administrators_see_it() { + let test_cases = [ + InstallationTestCase { + description: "SQLite", + backend: Backend::Sqlite, + expected_storage: StorageKind::Sqlite, + }, + InstallationTestCase { + description: "PostgreSQL", + backend: Backend::Postgres, + expected_storage: StorageKind::Postgresql, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store) + .await + .expect("migrate installation store"); + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("administrator"); + let developer = create_principal(store, &context, "orders-developer") + .await + .expect("developer"); + let installation = read_installation(store, (&context, admin)) + .await + .expect("administrator reads the installation"); + assert_eq!( + ( + installation.storage, + installation.schema_checksum.len(), + installation.schema_applied_at == installation.installed_at + ), + (test_case.expected_storage, 64, true), + "{}: the storage, a SHA-256 schema digest, and a schema created with the installation", + test_case.description + ); + assert!( + matches!( + read_installation(store, (&context, developer)).await, + Err(ControlError::Denied { .. }) + ), + "{}: a person without user.manage everywhere is refused", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/platform/test_problems.rs b/crates/sqlbuild-orchestration/tests/platform/test_problems.rs index 3ebb270633..6e2ed8a3a8 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_problems.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_problems.rs @@ -3,9 +3,14 @@ use crate::authorization::main::{ }; use crate::credentials::main::identities::create_principal::create_principal; use crate::credentials::models::{AuditContext, BuiltinRole, DefaultRole, RoleAssignmentRequest}; +use crate::platform::_helpers::observations::observe; +use crate::platform::main::list_person_platform_events::list_person_platform_events; use crate::platform::main::read_person_problems::read_person_problems; use crate::platform::main::report_location_health::report_location_health; -use crate::platform::models::{LocationHealth, LocationHealthReport}; +use crate::platform::models::{ + LocationHealth, LocationHealthReport, PersonPlatformEvent, PersonPlatformEventPage, + PlatformEvent, PlatformEventKind, +}; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::main::begin::begin; @@ -138,6 +143,22 @@ fn given_broken_locations_and_late_ticks_when_people_read_problems_then_only_lis 200, "each listed target keeps its own 100 late ticks" ); + assert_eq!( + everything + .low_calendars + .iter() + .map(|calendar| ( + calendar.location_name.as_str(), + calendar.name.as_str(), + calendar.horizon_end.is_some() + )) + .collect::>(), + vec![ + ("inventory", "expired_orders", true), + ("orders", "expired_orders", true) + ], + "only running calendars whose dates run out within a day, not paused or ten-day ones" + ); let scoped = read(developer).await.expect("developer problems"); assert_eq!(scoped.scheduler_leader, everything.scheduler_leader); assert_eq!( @@ -150,6 +171,15 @@ fn given_broken_locations_and_late_ticks_when_people_read_problems_then_only_lis "only the listed location is reported broken" ); assert_eq!(scoped.overdue_ticks.len(), 100); + assert_eq!( + scoped + .low_calendars + .iter() + .map(|calendar| calendar.location_id) + .collect::>(), + vec![locations[1]], + "another location's low calendar stays hidden" + ); assert!( scoped .overdue_ticks @@ -160,7 +190,134 @@ fn given_broken_locations_and_late_ticks_when_people_read_problems_then_only_lis ); let hidden = read(outsider).await.expect("outsider problems"); assert_eq!(hidden.scheduler_leader, everything.scheduler_leader); - assert!(hidden.broken_locations.is_empty() && hidden.overdue_ticks.is_empty()); + assert!( + hidden.broken_locations.is_empty() + && hidden.overdue_ticks.is_empty() + && hidden.low_calendars.is_empty() + ); + let transaction = begin(store, TransactionMode::Write) + .await + .expect("observe events"); + let (mut transaction, rows) = query( + transaction, + "SELECT trigger_id FROM sqb_triggers WHERE location_id=$1 LIMIT 1", + vec![Parameter::Id(locations[0])], + ) + .await + .expect("orders trigger"); + let trigger_id: Uuid = + serde_json::from_value(rows[0]["trigger_id"].clone()).expect("trigger ID"); + let now = transaction.now; + for event in [ + PlatformEvent::SchedulerNoLeader, + PlatformEvent::Heartbeat { + replica_id: Uuid::now_v7(), + }, + PlatformEvent::LocationBroken { + location_id: locations[1], + target: "development".into(), + last_seen_at: now, + }, + PlatformEvent::TickOverdue { + trigger_id, + scheduled_for: now, + }, + ] { + transaction = observe(transaction, Uuid::nil(), (event, Some(now))) + .await + .expect("observe"); + } + commit(transaction).await.expect("observed"); + let history = |principal: Uuid, page: (Option, u32)| { + list_person_platform_events(store, principal, page, DefaultRole::None) + }; + let kinds = |events: &[PersonPlatformEvent]| { + events + .iter() + .map(|event| event.event.kind()) + .collect::>() + }; + let all = history(admin, (None, 100)) + .await + .expect("administrator events") + .events; + assert_eq!( + kinds(&all), + [ + PlatformEventKind::TickOverdue, + PlatformEventKind::LocationBroken, + PlatformEventKind::SchedulerNoLeader + ], + "{}: newest first, without heartbeats", + test_case.description + ); + assert_eq!( + (all[0].location_id, all[0].target.as_deref()), + (Some(locations[0]), Some("development")), + "a tick's event names its trigger's location target" + ); + let first = history(admin, (None, 1)).await.expect("first page"); + let second = history(admin, (first.before, 1)) + .await + .expect("second page"); + assert_eq!(kinds(&second.events), [PlatformEventKind::LocationBroken]); + assert_eq!( + history(developer, (None, 1)) + .await + .expect("developer's first page"), + PersonPlatformEventPage { + events: Vec::new(), + before: Some(all[0].storage_id) + }, + "a page whose events are all hidden still continues the scan" + ); + let hidden = history(developer, (None, 1)) + .await + .expect("developer's first page"); + let broken = history(developer, (hidden.before, 1)) + .await + .expect("developer's second page"); + let no_leader = history(developer, (broken.before, 1)) + .await + .expect("developer's third page"); + let end = history(developer, (no_leader.before, 1)) + .await + .expect("developer's last page"); + assert_eq!( + (kinds(&broken.events), kinds(&no_leader.events), end), + ( + vec![PlatformEventKind::LocationBroken], + vec![PlatformEventKind::SchedulerNoLeader], + PersonPlatformEventPage { + events: Vec::new(), + before: None + } + ), + "paging through hidden events reaches older visible ones and then ends" + ); + assert_eq!( + kinds( + &history(developer, (None, 100)) + .await + .expect("developer events") + .events + ), + [ + PlatformEventKind::LocationBroken, + PlatformEventKind::SchedulerNoLeader + ], + "another location's tick stays hidden" + ); + assert_eq!( + kinds( + &history(outsider, (None, 100)) + .await + .expect("outsider events") + .events + ), + [PlatformEventKind::SchedulerNoLeader], + "only server-wide events without a listed target" + ); close_test_store(fixture).await; }); } diff --git a/crates/sqlbuild-orchestration/tests/platform/test_types.rs b/crates/sqlbuild-orchestration/tests/platform/test_types.rs index 8fa6dd4a47..9b0dcf7bb7 100644 --- a/crates/sqlbuild-orchestration/tests/platform/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/platform/test_types.rs @@ -1,6 +1,6 @@ use std::sync::{Arc, Mutex}; -use crate::platform::models::{PlatformEvent, PlatformEventEnvelope}; +use crate::platform::models::{PlatformEvent, PlatformEventEnvelope, StorageKind}; use chrono::{DateTime, Utc}; use serde::Deserialize; @@ -58,3 +58,9 @@ pub(crate) struct ProblemsTestCase { pub(crate) leaders: usize, pub(crate) expected_leader: bool, } + +pub(crate) struct InstallationTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_storage: StorageKind, +} diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index a19bd5068d..b8aab47a25 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -93,6 +93,7 @@ pub(crate) fn request( settings: BTreeMap::new(), window: None, concurrency_groups: BTreeSet::new(), + narrowing: None, } } diff --git a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs index b9aefe9a36..270ea19c73 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_claims.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_claims.rs @@ -12,12 +12,18 @@ use crate::store::tests::helpers::read_recall_events; use uuid::Uuid; use crate::credentials::tests::location_helpers::create_location; +use crate::runs::_helpers::lifecycle::reads::list_runs_in; +use crate::runs::models::{QueueWait, RunQueueState}; use crate::runs::tests::claim_helpers::succeed; use crate::runs::tests::helpers::{queue, request, snapshot}; use crate::runs::tests::terminal_helpers::finish; use crate::runs::tests::test_types::{ - BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase, ScopedClaimTestCase, + BacklogClaimTestCase, GroupLimitTestCase, PriorityClaimsTestCase, QueueStateTestCase, + ScopedClaimTestCase, }; +use crate::runs::types::RunsListing; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::TransactionMode; use crate::store::tests::lease_helpers::extend_lease_through_test; use crate::store::tests::replica_helpers::second_store; use crate::store::tests::store_helpers::{close_store, close_test_store, open_test_store, run}; @@ -527,3 +533,130 @@ fn given_a_blocked_multi_limit_waiter_when_younger_work_arrives_then_shared_limi }); } } + +#[test] +fn given_claimed_work_here_and_elsewhere_when_listing_the_queue_then_each_run_shows_its_place_and_what_holds_it_back_without_naming_other_scopes() + { + let test_cases = [ + QueueStateTestCase { + description: "SQLite queue read mirrors admission", + backend: Backend::Sqlite, + expected_claim_order: [4, 3, 5], + }, + QueueStateTestCase { + description: "PostgreSQL queue read mirrors admission", + backend: Backend::Postgres, + expected_claim_order: [4, 3, 5], + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let other = create_location(store, "returns", "test-admin") + .await + .expect("other location"); + let here = publish_snapshot(store, &snapshot(location)) + .await + .expect("snapshot"); + let there = publish_snapshot(store, &snapshot(other)) + .await + .expect("other snapshot"); + let limits = + BTreeMap::from([("warehouse".to_owned(), NonZeroU32::new(2).expect("limit"))]); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let mut running = request((location, here.snapshot_id), 1, 0, &["orders"]); + running.concurrency_groups.insert("warehouse".into()); + let mut elsewhere = request((other, there.snapshot_id), 2, 0, &["products"]); + elsewhere.concurrency_groups.insert("warehouse".into()); + for input in [&running, &elsewhere] { + queue(store, input).await; + claim_next_run(store, &lease, &limits, None) + .await + .expect("claim") + .expect("claimable"); + } + let mut blocked = request((location, here.snapshot_id), 3, 0, &["customers", "orders"]); + blocked.concurrency_groups.insert("warehouse".into()); + let mut urgent = request((location, here.snapshot_id), 4, 5, &["inventory"]); + urgent.concurrency_groups.insert("warehouse".into()); + let free = request((location, here.snapshot_id), 5, 0, &["products"]); + for input in [&blocked, &urgent, &free] { + queue(store, input).await; + } + let transaction = begin(store, TransactionMode::Read).await.expect("read"); + let (transaction, listed) = list_runs_in( + transaction, + (location, "development"), + RunsListing::Queue(100), + ) + .await + .expect("queue"); + commit(transaction).await.expect("commit"); + let slot = QueueWait::ConcurrencyLimit { + limit: "warehouse".into(), + holders: vec![running.run_id], + elsewhere: 1, + }; + let expected = [ + ( + urgent.run_id, + RunQueueState { + position: 1, + waits: vec![slot.clone()], + }, + ), + ( + blocked.run_id, + RunQueueState { + position: 2, + waits: vec![ + QueueWait::WriteClaim { + run_id: Some(running.run_id), + node_key: "orders".into(), + other_nodes: 0, + }, + slot, + QueueWait::EarlierRun { + run_id: Some(urgent.run_id), + limit: "warehouse".into(), + }, + ], + }, + ), + ( + free.run_id, + RunQueueState { + position: 3, + waits: vec![QueueWait::WriteClaim { + run_id: None, + node_key: "products".into(), + other_nodes: 0, + }], + }, + ), + ]; + assert_eq!( + listed.iter().map(|run| run.run_id).collect::>(), + test_case.expected_claim_order.map(Uuid::from_u128), + "{}", + test_case.description + ); + let actual: Vec<_> = listed + .into_iter() + .map(|run| (run.run_id, run.queue.expect("queue state"))) + .collect(); + assert_eq!(actual, expected, "{}", test_case.description); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs index d4ef1c1b64..04088727a6 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -22,6 +22,8 @@ mod test_group_admission; mod test_groups; #[path = "test_job_detail.rs"] mod test_job_detail; +#[path = "test_job_subsets.rs"] +mod test_job_subsets; #[path = "test_lifecycle.rs"] mod test_lifecycle; #[path = "test_monitor.rs"] diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs index 84e674cfa8..6f90bc7572 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -83,7 +83,7 @@ fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queu queue(store, &physical).await; request_job_run( store, - &actor, + (&actor, None), &JobRunRequest { run_id: Uuid::from_u128(100), location_id: location, @@ -93,6 +93,8 @@ fn given_mixed_physical_and_parent_waiters_when_admitting_then_priority_and_queu priority: test_case.parent_priority, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }, ) .await @@ -183,7 +185,7 @@ fn given_group_children_sharing_a_limit_of_one_when_claiming_then_they_run_one_a publish_snapshot(store, &input).await.expect("publish"); request_job_run( store, - &actor, + (&actor, None), &JobRunRequest { run_id: Uuid::now_v7(), location_id: location, @@ -193,6 +195,8 @@ fn given_group_children_sharing_a_limit_of_one_when_claiming_then_they_run_one_a priority: 1, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }, ) .await @@ -285,7 +289,7 @@ fn given_an_unserved_queued_group_when_dispatching_an_eligible_scope_then_unknow publish_snapshot(store, &input).await.expect("publish"); request_job_run( store, - &actor, + (&actor, None), &JobRunRequest { run_id: location, location_id: location, @@ -295,6 +299,8 @@ fn given_an_unserved_queued_group_when_dispatching_an_eligible_scope_then_unknow priority: 1, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }, ) .await @@ -522,7 +528,7 @@ fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_th for index in 0..test_case.parents.len() { request_job_run( store, - &actor, + (&actor, None), &JobRunRequest { // Higher-priority arrivals contend with groups that already hold their limits. run_id: Uuid::from_u128(1000 - index as u128), @@ -533,6 +539,8 @@ fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_th priority: index as i64 + 1, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }, ) .await @@ -659,8 +667,10 @@ fn given_an_older_blocked_allow_group_when_an_independent_generation_is_queued_t priority: 1, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }; - request_job_run(store, &actor, &older) + request_job_run(store, (&actor, None), &older) .await .expect("older request"); input.generation = 2; @@ -678,7 +688,7 @@ fn given_an_older_blocked_allow_group_when_an_independent_generation_is_queued_t reason: "Independent replacement generation".into(), ..older.clone() }; - request_job_run(store, &actor, &newer) + request_job_run(store, (&actor, None), &newer) .await .expect("newer request"); let mut physical = request((location, first.snapshot_id), 3000, 10, &["products"]); @@ -825,7 +835,7 @@ fn given_a_large_healthy_project_with_blocked_groups_when_claiming_then_unrelate for index in 0..test_case.group_count { request_job_run( store, - &actor, + (&actor, None), &JobRunRequest { run_id: Uuid::from_u128(1000 + index as u128), location_id: location, @@ -835,6 +845,8 @@ fn given_a_large_healthy_project_with_blocked_groups_when_claiming_then_unrelate priority: 1, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }, ) .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs index 4911455b32..cef10c013d 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -80,14 +80,16 @@ fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_a priority: 1, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }; assert!( - request_job_run(store, &context, &request) + request_job_run(store, (&context, None), &request) .await .expect("request group") ); assert!( - !request_job_run(store, &context, &request) + !request_job_run(store, (&context, None), &request) .await .expect("retry group") ); @@ -226,7 +228,7 @@ fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_a job_key: "staging".into(), ..request.clone() }; - request_job_run(store, &context, &standalone) + request_job_run(store, (&context, None), &standalone) .await .expect("standalone request"); assert!( @@ -346,7 +348,7 @@ fn given_opposing_parent_and_child_limits_when_advancing_groups_then_both_groups for job_key in ["orders_group", "inventory_group"] { request_job_run( store, - &context, + (&context, None), &JobRunRequest { run_id: Uuid::now_v7(), location_id: location, @@ -356,6 +358,8 @@ fn given_opposing_parent_and_child_limits_when_advancing_groups_then_both_groups priority: 1, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }, ) .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs index eac63a861d..11b8b13088 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs @@ -17,7 +17,7 @@ use crate::runs::main::{ }; use crate::runs::models::{JobRunRequest, PersonRunRequest, RunRecord, RunRequest, RunWindow}; use crate::runs::tests::helpers::{json_column, rows, snapshot}; -use crate::runs::tests::test_types::JobDetailTestCase; +use crate::runs::tests::test_types::{ExactKeyPlanner, JobDetailTestCase}; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; @@ -63,7 +63,9 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r "defer_to": "/srv/orders/state", "defer_sources_to": "production", "warehouse": "postgres://orders:orders-db-pass@warehouse.example.test/orders", - "concurrency": 4}}, + "concurrency": 4}, + "select": "orders", "targets": ["development"], + "declared_at": {"path": "orchestration/jobs.py", "line": 7}}, "customers": {"description": "Build customers", "selection": ["customers"]}, })); input.documents.insert( @@ -76,7 +78,14 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r .nodes .get_mut("orders") .expect("orders node") - .definition = json!({"materialization": "table", "config": {"critical": true}}); + .definition = json!({"materialization": "table", + "config": {"critical": true, "cursor_type": "integer"}}); + input + .nodes + .get_mut("customers") + .expect("customers node") + .definition = + json!({"materialization": "table", "config": {"cursor_type": "timestamp"}}); publish_snapshot(store, &input).await.expect("publish jobs"); let administrator = (&roles.context, credentials(&roles.administrator)); let location = roles.location_id; @@ -111,10 +120,39 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r "{}: vars and a deferral path are withheld, a target name shows, URI passwords are redacted", test_case.description ); + assert_eq!( + ( + job.select.as_deref(), + job.exclude.as_deref(), + json!(job.targets), + json!(job.declared_at), + job.cursor_type.as_deref() + ), + ( + Some("orders"), + None, + json!(["development"]), + json!({"path": "orchestration/jobs.py", "line": 7}), + Some("integer") + ), + "{}: the selector as written, its targets, where it is declared and its nodes' cursor type", + test_case.description + ); let group = read(administrator, "orders_group") .await .expect("administrator") .expect("published group"); + assert_eq!( + ( + group.select, + json!(group.targets), + group.declared_at, + group.cursor_type + ), + (None, json!([]), None, None), + "{}: a snapshot without authored text reads as before, and mixed cursor types give none", + test_case.description + ); assert_eq!( group .members @@ -169,6 +207,8 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r priority: 0, window: None, full_refresh: true, + select: Vec::new(), + exclude: Vec::new(), }; let refused = launch("daily_orders"); assert!( @@ -176,7 +216,7 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r request_person_run( store, (&roles.context, operator), - PersonRunRequest::Job(&refused) + PersonRunRequest::Job(&ExactKeyPlanner, &refused) ) .await, Err(ControlError::Denied { .. }) @@ -190,7 +230,12 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r }; assert!( matches!( - request_person_run(store, administrator, PersonRunRequest::Job(&both)).await, + request_person_run( + store, + administrator, + PersonRunRequest::Job(&ExactKeyPlanner, &both) + ) + .await, Err(ControlError::Invalid(_)) ), "{}: a window and a full refresh are refused together", @@ -200,9 +245,13 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r let full_job = launch("daily_orders"); for request in [full_job.clone(), group_launch.clone()] { assert!( - request_person_run(store, administrator, PersonRunRequest::Job(&request)) - .await - .expect("administrator launch"), + request_person_run( + store, + administrator, + PersonRunRequest::Job(&ExactKeyPlanner, &request) + ) + .await + .expect("administrator launch"), "{}: an administrator may rebuild from scratch", test_case.description ); @@ -223,9 +272,13 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r full_refresh: false, ..launch("customers") }; - request_person_run(store, administrator, PersonRunRequest::Job(&plain)) - .await - .expect("ordinary launch"); + request_person_run( + store, + administrator, + PersonRunRequest::Job(&ExactKeyPlanner, &plain), + ) + .await + .expect("ordinary launch"); let lease = acquire_lease(store, Uuid::now_v7()) .await .expect("elect") diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs new file mode 100644 index 0000000000..34c162e5b1 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs @@ -0,0 +1,542 @@ +//! A narrowed job run keeps only the job's own nodes, stays a run of that job, writes only what it selects and needs view on every node. + +use std::collections::BTreeSet; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::credentials::tests::location_helpers::create_location; +use crate::credentials::tests::role_helpers::{ + SUPPORT_ROLE, credentials, orders_grant, role_fixture, +}; +use crate::runs::main::change_run::change_run; +use crate::runs::main::list_person_runs::list_person_runs; +use crate::runs::main::list_runs::list_runs; +use crate::runs::main::request_job_run::request_job_run; +use crate::runs::main::request_person_run::request_person_run; +use crate::runs::main::request_rerun::request_rerun; +use crate::runs::models::{ + JobRunRequest, PersonRunRequest, RerunRequest, RunAuthority, RunChange, RunChangeRequest, + RunKind, RunNarrowing, RunRequest, +}; +use crate::runs::tests::helpers::{json_column, rows, snapshot}; +use crate::runs::tests::test_types::{ + ExactKeyPlanner, JobSubsetTestCase, NamedRerunTestCase, NarrowingDenialTestCase, +}; +use crate::runs::types::RunsListing; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::models::Parameter; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_published_job_when_a_run_narrows_it_then_only_the_jobs_own_selected_nodes_are_frozen() { + let test_cases = [ + JobSubsetTestCase { + description: "an exclusion leaves the job's other nodes", + job_key: "daily_orders", + select: &[], + exclude: &["inventory"], + expected_selection: Some(&["customers", "orders"]), + }, + JobSubsetTestCase { + description: "a selection outside the job adds nothing", + job_key: "daily_orders", + select: &["orders", "products"], + exclude: &[], + expected_selection: Some(&["orders"]), + }, + JobSubsetTestCase { + description: "a job selector excludes another job's nodes", + job_key: "daily_orders", + select: &[], + exclude: &["job:customer_sync"], + expected_selection: Some(&["inventory", "orders"]), + }, + JobSubsetTestCase { + description: "a selection that leaves none of the job's nodes is refused", + job_key: "daily_orders", + select: &["products"], + exclude: &[], + expected_selection: None, + }, + JobSubsetTestCase { + description: "a flag cannot ride in as a selector", + job_key: "daily_orders", + select: &["orders", "--full-refresh"], + exclude: &[], + expected_selection: None, + }, + JobSubsetTestCase { + description: "an unknown selector is the planner's refusal", + job_key: "daily_orders", + select: &[], + exclude: &["shipments"], + expected_selection: None, + }, + JobSubsetTestCase { + description: "a job group cannot be narrowed", + job_key: "nightly", + select: &[], + exclude: &["inventory"], + expected_selection: None, + }, + ]; + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location_id = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location_id); + input.documents.insert("jobs".into(), json!({ + "daily_orders": {"description": "Orders and stock", "selection": ["customers", "inventory", "orders"], + "concurrency_groups": ["orders-load"]}, + "customer_sync": {"description": "Customers", "selection": ["customers"]}, + })); + input.documents.insert( + "groups".into(), + json!({"nightly": { + "description": "Both jobs", "selection": ["customers", "inventory", "orders"], + "jobs": {"customer_sync": ["daily_orders"], "daily_orders": []}}}), + ); + publish_snapshot(store, &input).await.expect("publish jobs"); + let strings = + |tokens: &[&str]| tokens.iter().map(|token| (*token).to_owned()).collect(); + let mut run_ids: Vec = Vec::new(); + for test_case in &test_cases { + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: test_case.job_key.into(), + reason: "Rebuild part of the job".into(), + priority: 0, + window: None, + full_refresh: false, + select: strings(test_case.select), + exclude: strings(test_case.exclude), + }; + let outcome = + request_job_run(store, (&context, Some(&ExactKeyPlanner)), &request).await; + let refused = matches!(outcome, Err(ControlError::Invalid(_))); + assert!( + refused || outcome.as_ref().is_ok_and(|applied| *applied), + "{backend:?}: {}: {outcome:?}", + test_case.description + ); + let stored = rows( + store, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await; + let frozen: Option = stored.first().map(|row| { + serde_json::from_value(json_column(&row["request_json"])).expect("frozen") + }); + let listed = list_runs(store, (location_id, "development"), 50) + .await + .expect("runs"); + assert!( + listed.iter().all(|run| run.window.is_none()), + "a run without a backfill window lists none" + ); + assert_eq!( + listed + .iter() + .find(|run| run.run_id == request.run_id) + .map(|run| run.narrowing.clone()), + test_case.expected_selection.map(|_| { + Some(RunNarrowing { + select: request.select.clone(), + exclude: request.exclude.clone(), + }) + }), + "{backend:?}: {}: the run lists the selectors it was narrowed with, and nothing else it was requested with", + test_case.description + ); + run_ids.push(request.run_id); + let keys = |keys: &[&str]| -> BTreeSet { + keys.iter().map(|key| (*key).to_owned()).collect() + }; + assert_eq!( + frozen.map(|frozen| ( + frozen.kind, + frozen.job_key, + frozen.selection, + frozen.write_set, + frozen.concurrency_groups, + )), + test_case.expected_selection.map(|expected| ( + RunKind::Job, + Some("daily_orders".to_owned()), + keys(expected), + keys(expected), + keys(&["orders-load"]), + )), + "{backend:?}: {}: a run of the job, writing only what it selects, or refused", + test_case.description + ); + } + let source_id = run_ids[0]; + let narrowing = RunNarrowing { + select: strings(test_cases[0].select), + exclude: strings(test_cases[0].exclude), + }; + change_run( + store, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: source_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: context.actor.clone(), + reason: "Stop before it starts".into(), + }, + }, + ) + .await + .expect("cancel the queued narrowed run"); + let rerun_id = Uuid::now_v7(); + request_rerun( + store, + (&context, None), + &RerunRequest { + run_id: rerun_id, + source_run_id: source_id, + from_failure: false, + reason: "Run the same part again".into(), + priority: 0, + nodes: Vec::new(), + }, + ) + .await + .expect("rerun"); + let listed = list_runs(store, (location_id, "development"), 50) + .await + .expect("runs"); + assert_eq!( + listed + .iter() + .find(|run| run.run_id == rerun_id) + .map(|run| run.narrowing.clone()), + Some(Some(narrowing)), + "{backend:?}: a rerun of part of a job is listed with the same narrowing" + ); + let unplanned = JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Rebuild part of the job".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: vec!["inventory".into()], + }; + assert!( + matches!( + request_job_run(store, (&context, None), &unplanned).await, + Err(ControlError::Invalid(_)) + ), + "{backend:?}: a caller without a planner cannot narrow a job" + ); + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_member_without_view_on_every_node_when_narrowing_a_job_then_every_selector_is_refused_alike() + { + let test_cases = [ + NarrowingDenialTestCase { + description: "a node the member may run", + select: &["orders"], + exclude: &[], + expected_denied: true, + }, + NarrowingDenialTestCase { + description: "a node that does not exist", + select: &[], + exclude: &["shipments"], + expected_denied: true, + }, + NarrowingDenialTestCase { + description: "a node outside the job", + select: &["products"], + exclude: &[], + expected_denied: true, + }, + NarrowingDenialTestCase { + description: "a job node without a grant", + select: &["customers"], + exclude: &[], + expected_denied: true, + }, + NarrowingDenialTestCase { + description: "a job selector", + select: &["job:daily_orders"], + exclude: &[], + expected_denied: true, + }, + ]; + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store).await; + let mut input = snapshot(roles.location_id); + input.generation = 2; + input.documents.insert( + "roles".into(), + json!({SUPPORT_ROLE: {"description": "Answer order questions", + "grants": [orders_grant(json!(["view", "job.run"]), &["orders"])]}}), + ); + input.documents.insert("jobs".into(), json!({"daily_orders": { + "description": "Orders and stock", "selection": ["customers", "inventory", "orders"]}})); + publish_snapshot(store, &input).await.expect("publish jobs"); + let member = (&roles.context, credentials(&roles.member)); + let strings = + |tokens: &[&str]| tokens.iter().map(|token| (*token).to_owned()).collect(); + for test_case in &test_cases { + let request = JobRunRequest { + run_id: Uuid::now_v7(), + location_id: roles.location_id, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Rebuild part of the job".into(), + priority: 0, + window: None, + full_refresh: false, + select: strings(test_case.select), + exclude: strings(test_case.exclude), + }; + let outcome = request_person_run( + store, + member, + PersonRunRequest::Job(&ExactKeyPlanner, &request), + ) + .await; + assert_eq!( + matches!(outcome, Err(ControlError::Denied { .. })), + test_case.expected_denied, + "{backend:?}: {}: denied before any selector resolves, so nothing about the nodes leaks: {outcome:?}", + test_case.description + ); + } + close_test_store(fixture).await; + }); + } +} + +#[test] +fn given_a_finished_run_when_rerunning_named_nodes_then_only_those_rerun_for_people_who_list_the_target() + { + let test_cases = [ + NamedRerunTestCase { + description: "two of the run's nodes", + nodes: &["orders", "inventory"], + from_failure: false, + by_member: false, + expected_selection: Some(&["inventory", "orders"]), + expected_denied: false, + }, + NamedRerunTestCase { + description: "a node outside the run", + nodes: &["orders", "products"], + from_failure: false, + by_member: false, + expected_selection: None, + expected_denied: false, + }, + NamedRerunTestCase { + description: "named nodes and from failure together", + nodes: &["orders"], + from_failure: true, + by_member: false, + expected_selection: None, + expected_denied: false, + }, + NamedRerunTestCase { + description: "a member without view on every node, naming a node they may rerun", + nodes: &["orders"], + from_failure: false, + by_member: true, + expected_selection: None, + expected_denied: true, + }, + NamedRerunTestCase { + description: "a member without view on every node, naming a node outside the run", + nodes: &["products"], + from_failure: false, + by_member: true, + expected_selection: None, + expected_denied: true, + }, + ]; + for backend in [Backend::Sqlite, Backend::Postgres] { + run(async { + let Some(fixture) = open_test_store(backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store).await; + let mut input = snapshot(roles.location_id); + input.generation = 2; + input.documents.insert( + "roles".into(), + json!({SUPPORT_ROLE: {"description": "Answer order questions", + "grants": [orders_grant(json!(["view", "run.rerun"]), &["orders"])]}}), + ); + input.documents.insert("jobs".into(), json!({"daily_orders": { + "description": "Orders and stock", "selection": ["customers", "inventory", "orders"]}})); + publish_snapshot(store, &input).await.expect("publish jobs"); + let owner = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let source_id = Uuid::now_v7(); + request_job_run( + store, + (&owner, None), + &JobRunRequest { + run_id: source_id, + location_id: roles.location_id, + target: "development".into(), + job_key: "daily_orders".into(), + reason: "Build orders and stock".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + }, + ) + .await + .expect("source run"); + change_run( + store, + RunAuthority::Operator(&owner), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: source_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: owner.actor.clone(), + reason: "Stop before it starts".into(), + }, + }, + ) + .await + .expect("finish the source run"); + let people = [ + (&roles.context, credentials(&roles.administrator)), + (&roles.context, credentials(&roles.member)), + ]; + for test_case in &test_cases { + let request = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: source_id, + from_failure: test_case.from_failure, + reason: "Rerun part of the run".into(), + priority: 0, + nodes: test_case + .nodes + .iter() + .map(|node| (*node).to_owned()) + .collect(), + }; + let (context, principal) = people[usize::from(test_case.by_member)]; + let outcome = request_rerun(store, (context, Some(principal)), &request).await; + let stored = rows( + store, + "SELECT request_json FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await; + let frozen: Option = stored.first().map(|row| { + serde_json::from_value(json_column(&row["request_json"])).expect("frozen") + }); + let keys = |keys: &[&str]| -> BTreeSet { + keys.iter().map(|key| (*key).to_owned()).collect() + }; + assert_eq!( + ( + frozen.map(|frozen| (frozen.selection, frozen.write_set)), + matches!(outcome, Err(ControlError::Denied { .. })), + ), + ( + test_case + .expected_selection + .map(|expected| (keys(expected), keys(expected))), + test_case.expected_denied, + ), + "{backend:?}: {}: {outcome:?}", + test_case.description + ); + } + let list = |before| { + list_person_runs( + store, + people[0], + ( + roles.location_id, + "development", + RunsListing::Newest(1, before), + ), + DefaultRole::None, + ) + }; + let everything = list_person_runs( + store, + people[0], + ( + roles.location_id, + "development", + RunsListing::Newest(1000, None), + ), + DefaultRole::None, + ) + .await + .expect("all runs"); + let mut paged: Vec = Vec::new(); + let mut before = None; + for _ in 0..everything.len() { + let page = list(before).await.expect("page"); + paged.extend(page.iter().map(|run| run.run_id)); + before = page.last().map(|run| (run.queued_at, run.run_id)); + } + assert_eq!( + (paged, everything.len()), + ( + everything.iter().map(|run| run.run_id).collect::>(), + 2 + ), + "{backend:?}: pages of one continue after the last run of the previous page, newest first" + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs index c47dc489cd..187b602e8a 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_permissions.rs @@ -26,7 +26,8 @@ use crate::runs::models::{ RunChangeRequest, }; use crate::runs::tests::helpers::{request_run, snapshot}; -use crate::runs::tests::test_types::PersonRunPermissionTestCase; +use crate::runs::tests::test_types::{ExactKeyPlanner, PersonRunPermissionTestCase}; +use crate::runs::types::RunsListing; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; @@ -141,13 +142,15 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi priority: 0, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }; assert!( matches!( request_person_run( store, (&context, credentials), - PersonRunRequest::Job(&request) + PersonRunRequest::Job(&ExactKeyPlanner, &request) ) .await, Err(ControlError::Denied { .. }) @@ -174,7 +177,7 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi list_person_runs( store, (&context, credentials), - (location_id, "development", 10), + (location_id, "development", RunsListing::Newest(10, None)), DefaultRole::None, ) .await @@ -190,11 +193,14 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi let result = request_person_run( store, (&context, credentials), - PersonRunRequest::Job(&JobRunRequest { - run_id: Uuid::now_v7(), - job_key: job_key.into(), - ..request.clone() - }), + PersonRunRequest::Job( + &ExactKeyPlanner, + &JobRunRequest { + run_id: Uuid::now_v7(), + job_key: job_key.into(), + ..request.clone() + }, + ), ) .await; assert_eq!( @@ -212,7 +218,7 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi let source_id = Uuid::now_v7(); request_job_run( store, - &context, + (&context, None), &JobRunRequest { run_id: source_id, job_key: job_key.into(), @@ -244,7 +250,7 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi let source_id = Uuid::now_v7(); request_job_run( store, - &context, + (&context, None), &JobRunRequest { run_id: source_id, job_key: job_key.into(), @@ -283,6 +289,7 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi from_failure: false, reason: "Rerun the original frozen execution".into(), priority: 0, + nodes: Vec::new(), }; for expected in [true, false] { let result = @@ -330,6 +337,7 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi from_failure: false, reason: "Repeat ad-hoc execution".into(), priority: 0, + nodes: Vec::new(), }; let can_rerun_adhoc = test_case.role == BuiltinRole::Admin || (test_case.role == BuiltinRole::Developer && settings_index == 0); @@ -390,10 +398,13 @@ fn given_scoped_person_roles_when_jobs_request_sensitive_settings_then_only_admi request_person_run( store, (&context, credentials), - PersonRunRequest::Job(&JobRunRequest { - run_id: Uuid::now_v7(), - ..request - }) + PersonRunRequest::Job( + &ExactKeyPlanner, + &JobRunRequest { + run_id: Uuid::now_v7(), + ..request + } + ) ) .await, Err(ControlError::Denied { .. }) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs index e5301db2af..eaec5f2d71 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_requests.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_requests.rs @@ -73,6 +73,8 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an end: "2026-07-04T01:17:23+05:30".parse().expect("end"), }), full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }; for invalid in [ JobRunRequest { @@ -103,7 +105,11 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an ..request.clone() }, ] { - assert!(request_job_run(store, &context, &invalid).await.is_err()); + assert!( + request_job_run(store, (&context, None), &invalid) + .await + .is_err() + ); } let fault = TransactionFault { description: "manual run audit failure", @@ -114,7 +120,11 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an postgres_body: "RAISE EXCEPTION 'injected audit failure'", }; install_fault(store, &fault).await; - assert!(request_job_run(store, &context, &request).await.is_err()); + assert!( + request_job_run(store, (&context, None), &request) + .await + .is_err() + ); remove_fault(store, &fault).await; assert!( list_runs(store, (location_id, "development"), 10) @@ -129,8 +139,8 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an .is_empty() ); let (first, retry) = tokio::join!( - request_job_run(store, &context, &request), - request_job_run(store, &context, &request) + request_job_run(store, (&context, None), &request), + request_job_run(store, (&context, None), &request) ); assert_ne!( first.expect("first request"), @@ -148,22 +158,30 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an json!({"full_refresh": false}); publish_snapshot(store, &input).await.expect("redeploy"); assert!( - !request_job_run(store, &context, &request) + !request_job_run(store, (&context, None), &request) .await .expect("retry keeps original snapshot") ); let mut changed = request.clone(); changed.reason = "Different request".into(); - assert!(request_job_run(store, &context, &changed).await.is_err()); + assert!( + request_job_run(store, (&context, None), &changed) + .await + .is_err() + ); changed = request.clone(); changed.window = Some(RunWindow::Integer { start: 3, end: 11 }); - assert!(request_job_run(store, &context, &changed).await.is_err()); + assert!( + request_job_run(store, (&context, None), &changed) + .await + .is_err() + ); let other_actor = AuditContext { actor: "other-operator".into(), ..context.clone() }; assert!( - request_job_run(store, &other_actor, &request) + request_job_run(store, (&other_actor, None), &request) .await .is_err() ); @@ -220,7 +238,7 @@ fn given_a_published_job_when_requested_and_retried_after_redeploy_then_audit_an ..request.clone() }; assert!( - request_job_run(store, &context, &fresh) + request_job_run(store, (&context, None), &fresh) .await .expect("new request uses redeploy") ); diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index 6d3856bd86..86490c97bd 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -72,7 +72,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ let source_id = Uuid::now_v7(); request_job_run( store, - &context, + (&context, None), &JobRunRequest { run_id: source_id, location_id: location, @@ -82,6 +82,8 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ priority: 1, window: Some(RunWindow::Integer { start: -3, end: 11 }), full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }, ) .await @@ -92,6 +94,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ from_failure: true, reason: "Retry failed orders".into(), priority: 7, + nodes: Vec::new(), }; assert!( request_rerun(store, (&context, None), &retry) @@ -319,18 +322,23 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ advance_groups(store, &lease, (&limits, scope)) .await .expect("finish rerun"); + let listed = list_runs(store, scope, 20) + .await + .expect("list finished reruns"); + let finished = listed + .iter() + .find(|run| run.run_id == request.run_id) + .expect("finished rerun exists"); assert_eq!( - list_runs(store, scope, 20) - .await - .expect("list finished reruns") - .iter() - .find(|run| run.run_id == request.run_id) - .expect("finished rerun exists") - .status, - test_case.expected_status, + finished.status, test_case.expected_status, "{}", test_case.description ); + assert_eq!( + finished.window, + Some(RunWindow::Integer { start: -3, end: 11 }), + "a rerun of a backfill is listed with its window" + ); assert!(slot_counts(store).await.is_empty()); assert!( !request_rerun(store, (&context, None), &request) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index d937ebc730..9d38196822 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -17,6 +17,12 @@ pub(crate) struct PersonRunPermissionTestCase { pub(crate) expected_allowed: bool, } +pub(crate) struct QueueStateTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_claim_order: [u128; 3], +} + pub(crate) struct ScopedClaimTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, @@ -163,35 +169,86 @@ impl crate::runs::types::CommandPlanner for ExactKeyPlanner { fn plan( &self, command: &[String], - resources: &[crate::runs::models::PlannerResource], + (resources, jobs): ( + &[crate::runs::models::PlannerResource], + &crate::runs::types::PublishedJobs, + ), ) -> Result { let mut plan = crate::runs::models::CommandPlan { verb: command.first().cloned().unwrap_or_default(), selection: Default::default(), settings: Default::default(), }; + let mut excluded: std::collections::BTreeSet = Default::default(); + let mut excluding = false; let mut tokens = command.iter().skip(1); while let Some(token) = tokens.next() { - match token.as_str() { + let keys: Vec = match token.as_str() { "--full-refresh" => { plan.settings .insert("full_refresh".into(), serde_json::json!(true)); + continue; } "--warehouse" => { let name = tokens.next().ok_or("--warehouse needs a name")?; plan.settings .insert("warehouse".into(), serde_json::json!(name)); + continue; } - key if resources.iter().any(|resource| resource.key == key) => { - plan.selection.insert(key.to_owned()); + "--select" | "--exclude" => { + excluding = token == "--exclude"; + continue; } - other => return Err(format!("{other} is not available for ad-hoc runs")), + key if resources.iter().any(|resource| resource.key == key) => vec![key.to_owned()], + other => match other.strip_prefix("job:").and_then(|name| jobs.get(name)) { + Some(keys) => keys.iter().cloned().collect(), + None => return Err(format!("{other} is not available for ad-hoc runs")), + }, + }; + if excluding { + excluded.extend(keys); + } else { + plan.selection.extend(keys); } } + if plan.selection.is_empty() && !excluded.is_empty() { + plan.selection = resources + .iter() + .map(|resource| resource.key.clone()) + .collect(); + } + plan.selection.retain(|key| !excluded.contains(key)); Ok(plan) } } +pub(crate) struct NarrowingDenialTestCase { + pub(crate) description: &'static str, + pub(crate) select: &'static [&'static str], + pub(crate) exclude: &'static [&'static str], + pub(crate) expected_denied: bool, +} + +pub(crate) struct JobSubsetTestCase { + pub(crate) description: &'static str, + pub(crate) job_key: &'static str, + pub(crate) select: &'static [&'static str], + pub(crate) exclude: &'static [&'static str], + /// The frozen selection, or `None` when the request is refused as invalid. + pub(crate) expected_selection: Option<&'static [&'static str]>, +} + +pub(crate) struct NamedRerunTestCase { + pub(crate) description: &'static str, + pub(crate) nodes: &'static [&'static str], + pub(crate) from_failure: bool, + /// The member holds view and run.rerun on `orders` only; otherwise the administrator asks. + pub(crate) by_member: bool, + /// The rerun's frozen selection, or `None` when it is refused. + pub(crate) expected_selection: Option<&'static [&'static str]>, + pub(crate) expected_denied: bool, +} + pub(crate) struct RunNodeErrorTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs index 32366c394a..997f1835e5 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/helpers.rs @@ -160,7 +160,7 @@ pub(crate) async fn finish_run( }; request_job_run( store, - &context, + (&context, None), &JobRunRequest { run_id, location_id: scope.0, @@ -170,6 +170,8 @@ pub(crate) async fn finish_run( priority: 10, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }, ) .await diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs index e97b015a8c..1148fac76e 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_contracts.rs @@ -10,6 +10,8 @@ mod test_delivery; mod test_execution_bounds; #[path = "test_nodes.rs"] mod test_nodes; +#[path = "test_person_state.rs"] +mod test_person_state; #[path = "test_recovery.rs"] mod test_recovery; #[path = "test_runs.rs"] diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs index 9d195c33c3..35b450e461 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_execution_bounds.rs @@ -96,7 +96,7 @@ fn given_a_wide_group_when_only_the_last_node_has_not_started_then_all_chunks_mu }; request_job_run( store, - &context, + (&context, None), &JobRunRequest { run_id: parent, location_id: location, @@ -106,6 +106,8 @@ fn given_a_wide_group_when_only_the_last_node_has_not_started_then_all_chunks_mu priority: 10, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }, ) .await diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs index 560cdbcef7..ea5a94d84f 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_nodes.rs @@ -168,7 +168,7 @@ fn given_any_run_rebuilds_when_consuming_then_only_genuine_scoped_successes_queu }; request_job_run( store, - &context, + (&context, None), &JobRunRequest { run_id, location_id: location, @@ -178,6 +178,8 @@ fn given_any_run_rebuilds_when_consuming_then_only_genuine_scoped_successes_queu priority: 10, window: None, full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), }, ) .await diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_person_state.rs b/crates/sqlbuild-orchestration/tests/sensors/test_person_state.rs new file mode 100644 index 0000000000..511f861b17 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/sensors/test_person_state.rs @@ -0,0 +1,273 @@ +//! A person reads a sensor's cursor, checks and run requests behind the listing gate; custom values pass through secret redaction. + +use std::time::Duration; + +use chrono::Utc; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::main::identities::create_principal::create_principal; +use crate::credentials::models::{BuiltinRole, DefaultRole, RoleAssignmentRequest}; +use crate::credentials::tests::role_helpers::{ + SUPPORT_ROLE, assign, credentials, orders_grant, role_fixture, +}; +use crate::declarations::main::list_triggers::list_triggers; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; +use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; +use crate::sensors::main::read_person_sensor::read_person_sensor; +use crate::sensors::models::PersonSensorCursor; +use crate::sensors::tests::helpers::{append_events, snapshot}; +use crate::sensors::tests::test_types::SensorTestCase; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::main::{begin::begin, commit::commit, query::query}; +use crate::store::models::{Parameter, TransactionMode}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_checked_sensor_when_people_read_its_state_then_listing_view_gates_it_and_custom_values_are_redacted() + { + let test_cases = [ + SensorTestCase { + description: "SQLite", + backend: Backend::Sqlite, + expected_runs: 50, + }, + SensorTestCase { + description: "PostgreSQL", + backend: Backend::Postgres, + expected_runs: 50, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store).await; + let mut input = snapshot(roles.location_id); + input.generation = 2; + input.documents.insert( + "roles".into(), + json!({SUPPORT_ROLE: {"description": "Answer order questions", + "grants": [orders_grant(json!(["view"]), &["orders"])]}}), + ); + publish_snapshot(store, &input) + .await + .expect("publish sensor"); + let trigger = list_triggers(store, roles.location_id, "development") + .await + .expect("triggers") + .remove(0); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + append_events(store, 0..105, Utc::now()).await; + let first = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + let first_runs = complete_builtin_sensor(store, &lease, &first) + .await + .expect("first page"); + tokio::time::sleep(Duration::from_secs(1)).await; + let second = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("next page") + .expect("due"); + let second_runs = complete_builtin_sensor(store, &lease, &second) + .await + .expect("second page"); + let administrator = (&roles.context, credentials(&roles.administrator)); + let read = |who, trigger_id, limit| { + read_person_sensor( + store, + who, + ((roles.location_id, "development"), (trigger_id, limit)), + DefaultRole::None, + ) + }; + + let state = read(administrator, trigger.trigger_id, 100) + .await + .expect("administrator") + .expect("published sensor"); + let checks = &state.evaluations; + assert_eq!( + ( + checks + .iter() + .map(|check| (check.evaluation_id, check.phase.as_str(), check.run_count)) + .collect::>(), + checks[0].cursor_before.event_storage_id, + checks[1].cursor_after.clone(), + state.cursor.clone(), + state.run_requests.len(), + ), + ( + vec![ + (second.evaluation_id, "completed", second_runs), + (first.evaluation_id, "completed", test_case.expected_runs), + ], + second.cursor.event_storage_id, + Some(checks[0].cursor_before.clone()), + checks[0].cursor_after.clone(), + test_case.expected_runs + second_runs, + ), + "{}: checks newest first, each starting where the last ended, with the runs they requested", + test_case.description + ); + assert!( + first_runs == test_case.expected_runs + && checks.iter().all(|check| check.finished_at.is_some() + && check.definition_fingerprint == state.definition_fingerprint), + "{}: both checks finished on the published definition", + test_case.description + ); + input.generation = 3; + input.documents.get_mut("triggers").expect("triggers")["health_orders"] + .as_object_mut() + .expect("sensor") + .extend([ + ("targets".into(), json!(["development"])), + ( + "declared_at".into(), + json!({"path": "orchestration/health.py", "line": 12}), + ), + ]); + publish_snapshot(store, &input) + .await + .expect("republish moved sensor"); + let moved = read(administrator, trigger.trigger_id, 1) + .await + .expect("administrator") + .expect("published sensor"); + let listed = list_triggers(store, roles.location_id, "development") + .await + .expect("triggers") + .remove(0); + assert_eq!( + ( + moved.definition_fingerprint, + json!(listed.definition_json)["declared_at"].clone(), + ), + ( + state.definition_fingerprint.clone(), + json!({"path": "orchestration/health.py", "line": 12}), + ), + "{}: where a sensor is written is published but does not change its definition", + test_case.description + ); + let newest = read(administrator, trigger.trigger_id, 1) + .await + .expect("administrator") + .expect("published sensor"); + assert_eq!( + ( + newest + .evaluations + .iter() + .map(|check| check.evaluation_id) + .collect::>(), + newest.run_requests.len() + ), + (vec![second.evaluation_id], 1), + "{}: the limit keeps the newest check and run request", + test_case.description + ); + + let transaction = begin(store, TransactionMode::Write).await.expect("write"); + let (transaction, _) = query( + transaction, + "UPDATE sqb_trigger_state SET cursor_json=$1 WHERE trigger_id=$2", + vec![ + Parameter::Json(json!({"event_storage_id": 7, + "custom": "postgres://orders:orders-db-pass@warehouse.example.test/orders"})), + Parameter::Id(trigger.trigger_id), + ], + ) + .await + .expect("custom cursor"); + commit(transaction).await.expect("commit"); + let custom = read(administrator, trigger.trigger_id, 1) + .await + .expect("administrator") + .expect("published sensor") + .cursor + .and_then(|cursor| cursor.custom); + assert_eq!( + custom.as_deref(), + Some("postgres://orders:[REDACTED]@warehouse.example.test/orders"), + "{}: a custom cursor passes through secret redaction", + test_case.description + ); + let viewer = create_principal(store, &roles.context, "orders-viewer") + .await + .expect("viewer"); + assign( + store, + (&roles.context, &roles.administrator), + RoleAssignmentRequest { + principal_id: viewer, + role: BuiltinRole::Viewer.into(), + location_id: None, + target: None, + revoke: false, + }, + ) + .await; + let viewed = read((&roles.context, viewer), trigger.trigger_id, 1) + .await + .expect("viewer lists the target") + .expect("published sensor"); + assert_eq!( + ( + viewed.cursor, + viewed.evaluations[0].cursor_after.clone(), + viewed + .run_requests + .iter() + .map(|request| request.run_key.as_str()) + .collect::>(), + ), + ( + Some(PersonSensorCursor { + event_storage_id: 7, + custom: None + }), + checks[0].cursor_after.clone(), + vec![""], + ), + "{}: view without view.evidence sees positions but not what the sensor computed", + test_case.description + ); + + assert!( + read(administrator, Uuid::now_v7(), 100) + .await + .expect("administrator") + .is_none(), + "{}: an unknown sensor is not found", + test_case.description + ); + let member = (&roles.context, roles.member_id); + for trigger_id in [trigger.trigger_id, Uuid::now_v7()] { + assert!( + matches!( + read(member, trigger_id, 100).await, + Err(ControlError::Denied { .. }) + ), + "{}: view on one node does not list the target, and refusal hides whether the sensor exists", + test_case.description + ); + } + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs index 1b88eb8d3b..388b01aeb5 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_runs.rs @@ -327,6 +327,7 @@ fn given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_r reason: "Retry only failed orders without borrowing earlier customer success" .into(), priority: 10, + nodes: Vec::new(), }; let context = AuditContext { actor: "test-admin".into(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 4b600d1dae..5f6e49b0bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -10,6 +10,7 @@ use crate::bindings::_helpers::clone::clone_command::run_clone_command; use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; use crate::bindings::_helpers::command_runtime::warehouses::validate_cli_warehouse; use crate::bindings::_helpers::compile_command::reuse_attempt::run_compile; +use crate::bindings::_helpers::compiled_project::job_catalog::with_job_scope; use crate::bindings::_helpers::contract_command::command::run_contract_command; use crate::bindings::_helpers::cost_command::command::run_cost_command; use crate::bindings::_helpers::dbt_command::command::run_dbt_command; @@ -466,7 +467,14 @@ fn routed_command<'py>( if let Some(failure) = failure { return Err(user_error(py, failure)); } - call_route(&invocation, route, handler, recall) + let project_dir: PathBuf = invocation.value(Source::EffectiveProjectDir)?.extract()?; + let selected_target: Option = match invocation.value(Source::Attr("target")) { + Ok(target) => optional_text(&target)?, + Err(_) => None, + }; + with_job_scope((project_dir, selected_target), || { + call_route(&invocation, route, handler, recall) + }) } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs new file mode 100644 index 0000000000..3cea848026 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs @@ -0,0 +1,93 @@ +//! Jobs for the command's target, as publication keeps them, loaded only when a selector names one. + +use std::borrow::Cow; +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; +use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; +use sqlbuild_analysis::graph::models::{JobSelection, ProjectGraph}; +use sqlbuild_cli::native_commands::main::effective_target::effective_target; + +use crate::bindings::_helpers::compiled_project::snapshot_declarations::discover_declarations; +use crate::bindings::_helpers::native_host::orchestration_declarations::job_selections; + +/// The running command's project directory, its `--target` and, once loaded, its jobs. +struct JobScope { + project_dir: PathBuf, + selected_target: Option, + catalog: Option>, +} + +thread_local! { + /// Per thread, since a command resolves its selectors on the thread that dispatches it. + static JOB_SCOPE: RefCell> = const { RefCell::new(None) }; +} + +/// Run one command with jobs selectable from `project_dir`, restoring the enclosing scope after. +pub(crate) fn with_job_scope( + (project_dir, selected_target): (PathBuf, Option), + run: impl FnOnce() -> T, +) -> T { + let previous = JOB_SCOPE.with(|scope| { + scope.replace(Some(JobScope { + project_dir, + selected_target, + catalog: None, + })) + }); + let result: T = run(); + JOB_SCOPE.with(|scope| *scope.borrow_mut() = previous); + result +} + +/// `graph` with the command's jobs attached when `selectors` name a job, otherwise `graph` itself. +pub(crate) fn graph_with_jobs<'g>( + py: Python<'_>, + graph: &'g ProjectGraph, + selectors: &[String], + also_known: &[&str], +) -> PyResult> { + if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { + return Ok(Cow::Borrowed(graph)); + } + Ok(match job_catalog(py)? { + Some(catalog) => Cow::Owned(attach_jobs(graph.clone(), catalog)), + None => Cow::Borrowed(graph), + }) +} + +/// The command's job and job group names, loaded on first use; empty outside a command. +pub(crate) fn job_names(py: Python<'_>) -> PyResult> { + Ok(job_catalog(py)? + .map(|catalog| catalog.into_keys().collect()) + .unwrap_or_default()) +} + +fn job_catalog(py: Python<'_>) -> PyResult>> { + let Some(scope) = JOB_SCOPE.with(|scope| { + scope.borrow().as_ref().map(|scope| match &scope.catalog { + Some(catalog) => Ok(catalog.clone()), + None => Err((scope.project_dir.clone(), scope.selected_target.clone())), + }) + }) else { + return Ok(None); + }; + let (project_dir, selected_target): (PathBuf, Option) = match scope { + Ok(catalog) => return Ok(Some(catalog)), + Err(scope) => scope, + }; + let target: Option = + effective_target(&project_dir, selected_target).map_err(PyValueError::new_err)?; + let roots: Vec<(Bound<'_, PyAny>, String)> = discover_declarations(py, &project_dir)?; + let catalog: BTreeMap = job_selections(py, &roots, target.as_deref())?; + JOB_SCOPE.with(|scope| { + if let Some(scope) = scope.borrow_mut().as_mut() { + scope.catalog = Some(catalog.clone()); + } + }); + Ok(Some(catalog)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 5949e9aa31..8f4b12ec83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod attachment_inputs; pub(crate) mod constants; +pub(crate) mod job_catalog; pub(crate) mod project; pub(crate) mod registration; pub(crate) mod rows; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 882e3e8ba4..c926c807f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -16,18 +16,18 @@ use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - Definitions, GroupDefinition, JobDefinition, TriggerDeclaration, TriggerDefinition, + DeclaredAt, Definitions, GroupDefinition, JobDefinition, TriggerDeclaration, TriggerDefinition, }; use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, module_jobs, + AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, declared_at, module_jobs, }; use crate::bindings::_helpers::native_host::orchestration_triggers::{ compile_calendars, compile_cron, compile_sensors, }; -/// Dedicated declaration modules are imported only in the code-location interpreter. +/// Dedicated declaration modules, imported by publication and by selectors that name a job. pub(crate) fn discover_declarations<'py>( py: Python<'py>, project_dir: &Path, @@ -118,10 +118,16 @@ pub(crate) fn compile_declarations( .into_iter() .map(|(kind, name)| format!("{kind}:{name}")) .collect(); + let declared_at: Option = declared_at(project_dir, job.declared_at)?; + let targets: BTreeSet = job.targets.into_iter().collect(); if let Some(schedule) = job.schedule { triggers.insert( job.name.clone(), - compile_cron(schedule.bind(py), &job.name, &job.description)?, + TriggerDefinition { + targets: targets.clone(), + declared_at: declared_at.clone(), + ..compile_cron(schedule.bind(py), &job.name, &job.description)? + }, ); } jobs.insert( @@ -131,6 +137,10 @@ pub(crate) fn compile_declarations( selection, settings, concurrency_groups: job.concurrency_groups.into_iter().collect(), + select: Some(job.select), + exclude: Some(job.exclude).filter(|exclude| !exclude.trim().is_empty()), + targets, + declared_at, }, ); } @@ -159,10 +169,16 @@ pub(crate) fn compile_declarations( .iter() .flat_map(|name| jobs[name].selection.iter().cloned()) .collect(); + let declared_at: Option = declared_at(project_dir, group.declared_at)?; + let targets: BTreeSet = group.targets.into_iter().collect(); if let Some(schedule) = group.schedule { triggers.insert( group.name.clone(), - compile_cron(schedule.bind(py), &group.name, &group.description)?, + TriggerDefinition { + targets: targets.clone(), + declared_at: declared_at.clone(), + ..compile_cron(schedule.bind(py), &group.name, &group.description)? + }, ); } groups.insert( @@ -173,6 +189,10 @@ pub(crate) fn compile_declarations( selection, settings: BTreeMap::new(), concurrency_groups: group.concurrency_groups.into_iter().collect(), + select: None, + exclude: None, + targets, + declared_at, }, jobs: dependencies, }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs index f31b913f53..ce3c261a40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs @@ -186,6 +186,7 @@ const E_SELECTOR_KIND: EnumSpec = EnumSpec { ("TAG", "tag"), ("PATH", "path"), ("TEST", "test"), + ("JOB", "job"), ], }; const E_PYTHON_RUN_PHASE: EnumSpec = EnumSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 63aa2fd6ee..1dd9d85d85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -17,6 +17,7 @@ use sqlbuild_analysis::python_selection::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compiled_project::job_catalog::graph_with_jobs; use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -180,10 +181,13 @@ pub(crate) fn resolve_python_sql_run_selection( let native: &Bound<'_, NativeProjectGraph> = project_graph.get().native(py); let sources: Vec<(String, Option)> = project_sources(project_graph.get().project(py))?; let nodes: PythonSelectionGraph = selection_graph(python_graph)?; + let node_names: Vec<&str> = nodes.nodes.iter().map(|node| node.name.as_str()).collect(); + let selectors: Vec = select.iter().chain(&exclude).cloned().collect(); + let graph = graph_with_jobs(py, native.get().graph(), &selectors, &node_names)?; let selection: RunSelection = compiler_guard(|| { Ok(run_selection( SqlSelectionGraph { - graph: native.get().graph(), + graph: &graph, sources: &sources, }, &nodes, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 2e9bc74fb7..69148b027d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -67,7 +67,7 @@ const GRAPH_VIEW_NAMES: [&str; 5] = [ "path_index", ]; /// Selector kinds that match a set of resources rather than a named anchor. -const SET_SELECTOR_KINDS: [&str; 2] = ["tag", "path"]; +const SET_SELECTOR_KINDS: [&str; 3] = ["tag", "path", "job"]; /// The relation graph lineage reads: a cached index, or a compiled graph with its adapter. enum LineageSource<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index c5354c4914..52957406f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -13,6 +13,8 @@ pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; pub(crate) const SENSOR_CONTEXT_PARAMETER: &str = "ctx"; pub(crate) const AUTHORING_MODULE: &str = "sqlbuild.orchestration_declarations.models.authoring"; +/// Frames of these modules construct declarations on the author's behalf. +pub(crate) const CONSTRUCTING_MODULES: [&str; 2] = [AUTHORING_MODULE, "dataclasses"]; /// Kinds named in the global unique-name check. pub(crate) const JOB_LABEL: &str = "job"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 87651dcd21..1c4dda6f85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -1,18 +1,21 @@ //! The public authoring records enter native compilation here, never scheduler execution. use std::collections::{BTreeMap, HashSet}; -use std::path::Path; +use std::path::{Component, Path}; -use pyo3::FromPyObject; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use pyo3::{FromPyObject, pyfunction}; use serde_json::Value; +use sqlbuild_analysis::graph::models::JobSelection; use sqlbuild_discovery::project_validation::models::OrchestrationNameFact; use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::declarations::models::DeclaredAt; use crate::bindings::_helpers::native_host::constants::{ - AUTHORING_MODULE, CALENDAR_LABEL, GROUP_LABEL, JOB_LABEL, RUN_EVENT_CONDITION, SENSOR_LABEL, + AUTHORING_MODULE, CALENDAR_LABEL, CONSTRUCTING_MODULES, GROUP_LABEL, JOB_LABEL, + RUN_EVENT_CONDITION, SENSOR_LABEL, }; use crate::bindings::_helpers::native_host::host::entry; @@ -29,6 +32,8 @@ pub(crate) struct AuthoredJob { pub(crate) concurrency_groups: Vec, pub(crate) schedule: Option>, pub(crate) targets: Vec, + #[pyo3(attribute("_declared_at"))] + pub(crate) declared_at: Option<(String, u32)>, } #[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] @@ -45,6 +50,8 @@ pub(crate) struct AuthoredGroup { pub(crate) concurrency_groups: Vec, pub(crate) schedule: Option>, pub(crate) targets: Vec, + #[pyo3(attribute("_declared_at"))] + pub(crate) declared_at: Option<(String, u32)>, } #[derive(FromPyObject)] @@ -73,6 +80,8 @@ pub(crate) struct AuthoredSensor { pub(crate) minimum_interval_seconds: Value, pub(crate) default_status: String, pub(crate) targets: Vec, + #[pyo3(attribute("_declared_at"))] + pub(crate) declared_at: Option<(String, u32)>, } pub(crate) enum AuthoredDeclaration { @@ -82,6 +91,57 @@ pub(crate) enum AuthoredDeclaration { Sensor(AuthoredSensor), } +/// The file and line of the author's code constructing a declaration; `None` for code without a source file. +#[pyfunction] +pub(crate) fn declaration_site(py: Python<'_>) -> PyResult> { + let mut frame = py.import("sys")?.call_method1("_getframe", (0,))?; + while !frame.is_none() { + let module: Option = frame + .getattr("f_globals")? + .cast::()? + .get_item("__name__")? + .map(|name| name.extract()) + .transpose()?; + if !module.is_some_and(|module| CONSTRUCTING_MODULES.contains(&module.as_str())) { + let file: String = frame.getattr("f_code")?.getattr("co_filename")?.extract()?; + let line: u32 = frame.getattr("f_lineno")?.extract()?; + return Ok((!file.starts_with('<')).then_some((file, line))); + } + frame = frame.getattr("f_back")?; + } + Ok(None) +} + +/// A declaration's file relative to the project; code outside the project keeps its location private. +pub(crate) fn declared_at( + project_dir: &Path, + written: Option<(String, u32)>, +) -> PyResult> { + let Some((file, line)) = written else { + return Ok(None); + }; + let file = Path::new(&file); + let roots = [ + project_dir.canonicalize()?, + std::path::absolute(project_dir)?, + ]; + let Some(root) = roots.iter().find(|root| file.starts_with(root)) else { + return Ok(None); + }; + let parts: Option> = file + .components() + .skip(root.components().count()) + .map(|component| match component { + Component::Normal(part) => part.to_str(), + _ => None, + }) + .collect(); + Ok(parts.map(|parts| DeclaredAt { + path: parts.join("/"), + line, + })) +} + /// JSON authoring values cross once; the interpreter rejects circular user containers safely. pub(crate) fn authored_setting(value: &Bound<'_, PyAny>) -> PyResult { let text: String = value @@ -235,11 +295,11 @@ pub(crate) fn collect_declarations<'py>( Ok(collected) } -/// Every named declaration reachable from `roots`, labelled with the module of the root that reaches it first. -pub(crate) fn orchestration_names( - py: Python<'_>, - roots: &[(Bound<'_, PyAny>, String)], -) -> PyResult> { +/// Every declaration reachable from `roots`, once each, with its kind and the module of the root that reaches it first. +pub(crate) fn reachable_declarations<'py>( + py: Python<'py>, + roots: &[(Bound<'py, PyAny>, String)], +) -> PyResult, &'static str, String)>> { let authoring = py.import(AUTHORING_MODULE)?; let classes = [ (authoring.getattr("Job")?, JOB_LABEL), @@ -248,9 +308,9 @@ pub(crate) fn orchestration_names( (authoring.getattr("CalendarSchedule")?, CALENDAR_LABEL), ]; let mut seen: HashSet = HashSet::new(); - let mut names: Vec = Vec::new(); + let mut reached: Vec<(Bound<'py, PyAny>, &'static str, String)> = Vec::new(); for (root, path) in roots { - let mut pending: Vec> = vec![root.clone()]; + let mut pending: Vec> = vec![root.clone()]; while let Some(declaration) = pending.pop() { if !seen.insert(declaration.as_ptr() as usize) { continue; @@ -263,21 +323,29 @@ pub(crate) fn orchestration_names( continue; }; pending.extend(nested_declarations(&declaration, kind)?); - let name: Option = if kind == CALENDAR_LABEL { - let function = declaration.getattr("function")?; - (!function.is_none()) - .then(|| function.getattr("__name__")?.extract()) - .transpose()? - } else { - Some(declaration.getattr("name")?.extract()?) - }; - names.extend(name.map(|name| OrchestrationNameFact { - name, - kind, - path: path.clone(), - })); + reached.push((declaration, kind, path.clone())); } } + Ok(reached) +} + +/// Every named declaration reachable from `roots`, labelled with the module of the root that reaches it first. +pub(crate) fn orchestration_names( + py: Python<'_>, + roots: &[(Bound<'_, PyAny>, String)], +) -> PyResult> { + let mut names: Vec = Vec::new(); + for (declaration, kind, path) in reachable_declarations(py, roots)? { + let name: Option = if kind == CALENDAR_LABEL { + let function = declaration.getattr("function")?; + (!function.is_none()) + .then(|| function.getattr("__name__")?.extract()) + .transpose()? + } else { + Some(declaration.getattr("name")?.extract()?) + }; + names.extend(name.map(|name| OrchestrationNameFact { name, kind, path })); + } Ok(names) } @@ -305,3 +373,52 @@ fn nested_declarations<'py>( _ => Vec::new(), }) } + +/// Each reachable job's own selector tokens and each group's member names for `target`, as publication reads them. +pub(crate) fn job_selections( + py: Python<'_>, + roots: &[(Bound<'_, PyAny>, String)], + target: Option<&str>, +) -> PyResult> { + let mut catalog: BTreeMap = BTreeMap::new(); + for (declaration, kind, _) in reachable_declarations(py, roots)? { + if kind != JOB_LABEL && kind != GROUP_LABEL { + continue; + } + let targets: Vec = declaration.getattr("targets")?.extract()?; + if !targets.is_empty() + && !target.is_some_and(|target| targets.iter().any(|name| name == target)) + { + continue; + } + let tokens = |field: &str| -> PyResult> { + Ok(declaration + .getattr(field)? + .extract::()? + .split_whitespace() + .map(str::to_owned) + .collect()) + }; + let selection: JobSelection = match kind { + JOB_LABEL => JobSelection::Job { + select: tokens("select")?, + exclude: tokens("exclude")?, + }, + GROUP_LABEL => JobSelection::Group { + members: declaration + .getattr("jobs")? + .try_iter()? + .map(|job| job?.getattr("name")?.extract()) + .collect::>()?, + }, + _ => continue, + }; + let name: String = declaration.getattr("name")?.extract()?; + if catalog.insert(name.clone(), selection).is_some() { + return Err(PyValueError::new_err(format!( + "job name '{name}' is declared twice" + ))); + } + } + Ok(catalog) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 583ae015e2..8779ef8608 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -28,7 +28,7 @@ use crate::bindings::_helpers::native_host::constants::{ }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredSensor, authored_setting, + AuthoredSensor, authored_setting, declared_at, }; use crate::bindings::_helpers::native_host::orchestration_environment::{ callback_code_hash, orchestration_code_hash, @@ -56,6 +56,8 @@ fn compile_trigger( .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, default_status: serde_json::from_value(Value::String(controls.default_status)) .map_err(|error| PyValueError::new_err(format!("default_status: {error}")))?, + targets: BTreeSet::new(), + declared_at: None, }) } @@ -236,6 +238,13 @@ pub(crate) fn compile_calendars( let schedule = calendar_times(calendar, &session, &by_name, start)?; let mut trigger = compile_trigger(calendar, &job_key, &description, schedule)?; trigger.starts_at.get_or_insert(start); + trigger.targets = calendar + .getattr("targets")? + .extract::>()? + .into_iter() + .collect(); + trigger.declared_at = + declared_at(project_dir, calendar.getattr("_declared_at")?.extract()?)?; definitions.triggers.insert(name, trigger); } Ok(definitions) @@ -295,6 +304,8 @@ pub(crate) fn compile_sensors( "when": condition, "minimum_interval_seconds": sensor.minimum_interval_seconds, "default_status": sensor.default_status, + "targets": sensor.targets, + "declared_at": declared_at(project_dir, sensor.declared_at)?, })) .map_err(|error| PyValueError::new_err(format!("sensor {}: {error}", sensor.name)))?; definitions.sensors.insert(sensor.name, definition); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs index 71da763363..95ebe934fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs @@ -1,12 +1,16 @@ -//! Register `_native.HOST_ENTRIES`, the host table the Python import test checks against. +//! Register `_native.HOST_ENTRIES`, the host table the import test checks, and host functions. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::types::PyTuple; +use pyo3::wrap_pyfunction; use sqlbuild_host::entries::main::host_entry_labels::host_entry_labels; +use crate::bindings::_helpers::native_host::orchestration_declarations::declaration_site; + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add( "HOST_ENTRIES", PyTuple::new(module.py(), host_entry_labels())?, - ) + )?; + module.add_function(wrap_pyfunction!(declaration_site, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs index 52a86f9aed..e3bde9f64e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs @@ -11,6 +11,7 @@ use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::{ParsedSelector, ProjectGraph}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; +use crate::bindings::_helpers::compiled_project::job_catalog::graph_with_jobs; use crate::bindings::_helpers::graph::algorithms::closure; use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; @@ -107,12 +108,10 @@ pub(crate) fn resolve_graph_keys<'py>( let exclude: Vec> = py_strings(exclude)?; let texts: Vec> = select.iter().chain(&exclude).cloned().collect(); let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph)))?; - resolve_selectors( - graph, - &carrier.send_all(&select)?, - &carrier.send_all(&exclude)?, - ) - .map_err(|error| carrier.error(error)) + let (select, exclude) = (carrier.send_all(&select)?, carrier.send_all(&exclude)?); + let all: Vec = select.iter().chain(&exclude).cloned().collect(); + let graph = graph_with_jobs(py, graph, &all, &[])?; + resolve_selectors(&graph, &select, &exclude).map_err(|error| carrier.error(error)) } fn view_graph(mappings: GraphViewMappings<'_, '_>) -> PyResult { @@ -183,10 +182,9 @@ pub(crate) fn resolve_project_selector_tokens<'py>( let graph: NativeProjectGraph = arguments_graph(&arguments)?; let texts: Vec> = py_strings(&arguments.value("selectors")?)?; let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph.graph())))?; - selected( - &carrier, - resolve_selector_tokens(graph.graph(), &carrier.send_all(&texts)?), - ) + let selectors: Vec = carrier.send_all(&texts)?; + let graph = graph_with_jobs(py, graph.graph(), &selectors, &[])?; + selected(&carrier, resolve_selector_tokens(&graph, &selectors)) } fn arguments_graph(arguments: &KeywordArguments<'_>) -> PyResult { @@ -220,10 +218,11 @@ pub(crate) fn match_project_selector<'py>( Ok(graph_names(graph.graph())) })?; let kind: String = native_field(parsed, "kind")?.str()?.extract()?; + let value: String = carrier.send(&value)?; + let graph = graph_with_jobs(py, graph.graph(), &[format!("{kind}:{value}")], &[])?; selected( &carrier, - match_selector(graph.graph(), &kind, &carrier.send(&value)?) - .map(|keys| keys.into_iter().collect()), + match_selector(&graph, &kind, &value).map(|keys| keys.into_iter().collect()), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs index 30e188ca01..8318cff685 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs @@ -5,17 +5,25 @@ use std::collections::BTreeSet; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyString, PyTuple}; +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_analysis::graph::models::ParsedSelector; use sqlbuild_analysis::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors as split; use sqlbuild_analysis::planner_selection::models::{ SelectorGroups, UnitTestSelectorNames, UnitTestSelectorSplit, }; +use crate::bindings::_helpers::compiled_project::job_catalog::job_names; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::{field as native_field, path_stem}; use crate::bindings::_helpers::records::ids::RecordId; +/// A bare selector's kind. +const NAME_SELECTOR_KIND: &str = "name"; +/// Characters that make a selector value a glob pattern. +const PATTERN_CHARACTERS: [char; 3] = ['*', '?', '[']; + const RESOURCE_ENTRY_FILES: [(&str, &str); 3] = [ ("schema_files", "model_entries"), ("schema_files", "seed_entries"), @@ -88,13 +96,31 @@ fn restored<'py>(carrier: &Carrier<'py>, texts: &[String]) -> PyResult bool { + selectors + .iter() + .flat_map(|selector| selector.split(char::is_whitespace)) + .flat_map(|token| token.split(',')) + .filter(|part| !part.is_empty()) + .any(|part| match parse_selector(part) { + Ok(ParsedSelector::Kind { kind, value, .. }) => { + kind == NAME_SELECTOR_KIND + && !value.contains(PATTERN_CHARACTERS) + && !names.tests.contains(&value) + && !names.resources.contains(&value) + } + _ => false, + }) +} + fn split_python<'py>( (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), discovered: &Bound<'py, PyAny>, accepts_unit_tests: bool, ) -> PyResult> { let py: Python<'py> = select.py(); - let names = UnitTestSelectorNames { + let mut names = UnitTestSelectorNames { tests: unit_test_names(discovered)?, resources: resource_names(discovered)?, }; @@ -106,6 +132,10 @@ fn split_python<'py>( })?; let (select, exclude): (Vec, Vec) = (carrier.send_all(&select)?, carrier.send_all(&exclude)?); + let sent: Vec = select.iter().chain(&exclude).cloned().collect(); + if accepts_unit_tests && names_unknown(&sent, &names) { + names.resources.extend(job_names(py)?); + } let split: UnitTestSelectorSplit = split( SelectorGroups { select: &select, diff --git a/fensu.toml b/fensu.toml index 5460b89789..b2ee2457c4 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1377,7 +1377,7 @@ path = "src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py" reason = "The SQL Server adapter intentionally keeps overrideable backend behavior on one class." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/location_work.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_checks.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_maintenance.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_node_sql.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/trigger_reads.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs"] +paths = ["crates/sqlbuild-orchestration/src/api/_helpers/resources/administrators.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/audit.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/location_health.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/location_work.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_checks.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_location_work.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_maintenance.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_node_sql.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_plans.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/run_reads.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/runs.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/snapshots.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/trigger_reads.rs", "crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs"] reason = "Each function is one utoipa-annotated axum route: OpenAPI needs one annotated handler per operation, and each body only checks the caller's scope and calls one store operation. Scope checks and store work live in their shared owners." [[dupes.allowlist]] @@ -1636,8 +1636,6 @@ allowed = [ "sqb_password_credentials.changed_at=Audit history read through audit exports: when a password last changed.", "sqb_platform_sink_deliveries.delivered_at=Audit history read through audit exports: when an event reached the platform sink.", "sqb_security_audit_deliveries.delivered_at=Audit history read through audit exports: when an event reached a security audit sink.", - "sqb_run_requests.evaluation_id=Reader comes with the operations UI: a sensor's run requests show the evaluation that made them.", - "sqb_write_claims=Read through its primary-key conflict: an insert that conflicts means another run already holds the node.", "sqb_warehouse_microbatches.execution_run_id=Reader comes with the UI epic (D132), which filters on it: the run that executed the batch.", "sqb_warehouse_microbatches.record_type=Reader comes with the UI epic (D132), which filters on it: the batch state.", "sqb_warehouse_migrations.decision=Reader comes with the UI epic (D132), which filters on it.", @@ -2951,3 +2949,7 @@ reason = "A relation's declared columns and a dbt manifest's string field are un [[dupes.allowlist]] paths = ["crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs", "crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs", "crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs"] reason = "Each is a short stack walk over a JSON syntax tree with its own stop and collect rule: cast_targets collects every cast's target type, branch_selects stops at each set-operation branch, and walked lists every object. A shared visitor would need a callback for each rule and save no logic." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs"] +reason = "Field lookups on unrelated value types in different crates: project configuration takes the first matching field of a built object, while rule headers take the last entry of an authored map so a repeated key overrides earlier ones." diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index db2d7e2b93..a37f7ae52b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -13,6 +13,7 @@ Online: https://sqlbuild.com/docs/cli/server/ - Grant roles to OIDC groups (beta) - Inspect published SQL - Run ad-hoc commands +- Plan builds - Define project roles Run the control API and scheduler without loading project Python: @@ -25,15 +26,19 @@ Use `sqb server --help` for service, authentication and launcher options. ## Upgrade -Upgrade the server first, then the locations and workers that connect to it. A worker newer than -its server can send a run report the server cannot read. The worker then stops with: +Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the +server after a location or worker has connected from a newer release; the server can't read what +they wrote. + +| Newer than the server | What happens | Fix | +| --- | --- | --- | +| Worker | The worker refuses the run and stops with the message below. | Upgrade the server, then rerun the run. | +| Location | The server refuses the location's publication with `API rejected the request with HTTP 400 Bad Request`. The previous snapshot stays current. | Upgrade the server, then restart the location so it publishes again. | ```text the control server is older than this worker and refused its run report; upgrade the server before its workers ``` -Upgrade the server to the worker's release or newer, then rerun the run. - ## Set retention Set how much history the server keeps: @@ -231,6 +236,39 @@ the reason `the requested warehouse is not configured for this target on this lo warehouse statement ran`, which the run page shows. A job's own `warehouse` setting is not checked this way. +## Plan builds + +Anyone with `view` on every node of a target can see what a `build` command would do before it +runs. The web UI's plan page does this for every Run button: the location serving the target runs +[`sqb plan --json`](plan.md) on the command's nodes against the warehouse, and the page +shows each model's action and reason, backfills, migrations and warnings. Running the command +still needs `run.adhoc`. + +Over the API, queue a plan, then read it until `result_json` is set: + +```json +POST /api/control/locations/{location_id}/targets/{target}/plans +{ "request_id": "0191f0aa-0000-7000-8000-0000000000a1", "command": ["build", "--select", "+orders"] } + +GET /api/control/locations/{location_id}/targets/{target}/plans/{request_id} +``` + +`result_json.outcome.kind` is `plan`, with the `sqb plan --json` document in `document`, or +`superseded` when the location's project files changed while it planned; plan again. A plan never +appears in Runs and changes nothing in the warehouse. + +| Limit | Value | +| --- | --- | +| Verbs | `build` only | +| Flags | Those accepted for an [ad-hoc run](#run-ad-hoc-commands); `--warehouse` and flags `sqb plan` doesn't take, such as `--fail-fast`, don't change the plan | +| Location | A `sqb location serve` for the target that reported in the last 90 seconds | +| Plans waiting | One per person and four per target | +| Expiry | 10 minutes | +| Plan size | 8 MiB | + +Only the person who asked can read a plan. Python source and dependency diffs are left out unless +they also have `view.sql` on every node, and secrets in warehouse error text are redacted. + ## Define project roles Put project roles in `roles.toml` at the project root. `sqb location publish` resolves each diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index 853d2f3c0f..cf563c84a9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -80,6 +80,26 @@ sqb build --select seed:waffle_types sqb build --select source:raw__orders ``` +### Job + +Select the nodes a job or job group declared under `orchestration/` builds. Job names are globally +unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: + +```bash +sqb build nightly_orders +sqb build --select job:nightly_orders --exclude order_totals +sqb build --select "job:nightly_orders,tag:finance" +``` + +A job selects its own `select` minus its own `exclude`. A job group selects the union of its jobs. +A job or job group declared with `targets` is selectable only when the command's target is one of +them, as on the server. +Job selectors compose with `+`, `,` and `--exclude` like any other selector. A job's own selection +cannot name another job, and path-between selectors cannot start or end at a job. + +SQLBuild imports the modules under `orchestration/` only when a selector names a job or a name no +model, source, seed or function has. + ### Unit test `sqb test` and `sqb build` select SQL unit tests by name. Test names are globally unique, so a bare diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 75d62f786e..13bd5eaaca 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1765,6 +1765,7 @@ class NativeOrchestrationWorker: traceback: TracebackType | None, ) -> None: ... +def declaration_site() -> tuple[str, int] | None: ... def build_orchestration_snapshot( project: CompiledProjectView, metadata_json: str, @@ -8981,6 +8982,7 @@ class SelectorKind(StrEnum): TAG = "tag" PATH = "path" TEST = "test" + JOB = "job" class PythonRunPhase(StrEnum): """Lifecycle-aware Python execution phase.""" diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index 9c5d72f044..a904d71243 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -7,6 +7,7 @@ from datetime import datetime from typing import Literal +from sqlbuild._native import declaration_site from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind @@ -33,6 +34,10 @@ class Job: concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) + _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "_declared_at", declaration_site()) @dataclass(frozen=True, eq=False) @@ -46,6 +51,10 @@ class JobGroup: concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) + _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "_declared_at", declaration_site()) @dataclass(frozen=True, kw_only=True) @@ -68,6 +77,10 @@ class CalendarSchedule: default_status: Literal["running", "paused"] = "running" targets: tuple[str, ...] = () function: Callable[..., Iterable[datetime]] | None = field(default=None, repr=False) + _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "_declared_at", declaration_site()) def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: return replace(self, function=function) @@ -138,6 +151,10 @@ class Sensor: function: Callable[..., Iterable[RunRequest] | SkipReason] | None = field( default=None, repr=False, kw_only=True ) + _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "_declared_at", declaration_site()) def __call__(self, function: Callable[..., Iterable[RunRequest] | SkipReason]) -> Sensor: return replace(self, function=function) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 658a49eb50..a0984474b5 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1373,3 +1373,13 @@ class RuleFailureManifestTestCase: description: str compile_args: tuple[str, ...] expected_artifacts: dict[str, object] + + +@dataclass(frozen=True) +class JobSelectorBuildTestCase: + description: str + build_args: tuple[str, ...] + expected_exit_code: int + expected_tables: frozenset[str] + expected_jobs_imported: bool + expected_fragment: str = "" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py new file mode 100644 index 0000000000..6b1d2d8bf9 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py @@ -0,0 +1,135 @@ +"""Jobs select their nodes on the command line like any other resource, loading only when named.""" + +from __future__ import annotations + +from pathlib import Path + +import duckdb +import pytest + +from sqlbuild._native import run_cli as main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import JobSelectorBuildTestCase + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "demo"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "demo.duckdb"\n\n[targets.dev]\n\n[targets.prod]\n' + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table, tags [staging]);\n" + "SELECT 1 AS order_id\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table, tags [finance]);\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "models/customers.sql": ( + "MODEL (description 'Customers.', materialized table);\nSELECT 7 AS customer_id\n" + ), + "orchestration/jobs.py": ( + "from pathlib import Path\n" + "from sqlbuild.orchestration import job, job_group\n" + "Path(__file__).parent.parent.joinpath('jobs_imported').write_text('yes')\n" + "nightly = job('nightly', description='Orders.', select='+orders')\n" + "crm = job('crm', description='Customers.', select='customers')\n" + "daily = job_group('daily', description='Everything.', jobs=[nightly, crm])\n" + "restock = job('restock', description='Prod only.', select='customers', targets=['prod'])\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + JobSelectorBuildTestCase( + description="a job prefix builds the job's nodes", + build_args=("--select", "job:nightly"), + expected_exit_code=0, + expected_tables=frozenset({"orders", "stg_orders"}), + expected_jobs_imported=True, + ), + JobSelectorBuildTestCase( + description="a bare job name needs no prefix", + build_args=("--select", "nightly"), + expected_exit_code=0, + expected_tables=frozenset({"orders", "stg_orders"}), + expected_jobs_imported=True, + ), + JobSelectorBuildTestCase( + description="a job group builds its jobs' nodes", + build_args=("--select", "job:daily"), + expected_exit_code=0, + expected_tables=frozenset({"customers", "orders", "stg_orders"}), + expected_jobs_imported=True, + ), + JobSelectorBuildTestCase( + description="a job intersects with a tag", + build_args=("--select", "job:daily,tag:staging"), + expected_exit_code=0, + expected_tables=frozenset({"stg_orders"}), + expected_jobs_imported=True, + ), + JobSelectorBuildTestCase( + description="a job can be excluded from a group", + build_args=("--select", "job:daily", "--exclude", "job:crm"), + expected_exit_code=0, + expected_tables=frozenset({"orders", "stg_orders"}), + expected_jobs_imported=True, + ), + JobSelectorBuildTestCase( + description="a job declared for another target is not selectable", + build_args=("--select", "job:restock"), + expected_exit_code=1, + expected_tables=frozenset(), + expected_jobs_imported=True, + expected_fragment="unknown job name 'restock'", + ), + JobSelectorBuildTestCase( + description="a job declared for the selected target is selectable", + build_args=("--target", "prod", "--select", "job:restock"), + expected_exit_code=0, + expected_tables=frozenset({"customers"}), + expected_jobs_imported=True, + ), + JobSelectorBuildTestCase( + description="model selectors never import orchestration modules", + build_args=("--select", "+orders"), + expected_exit_code=0, + expected_tables=frozenset({"orders", "stg_orders"}), + expected_jobs_imported=False, + ), + JobSelectorBuildTestCase( + description="an unknown job suggests a known one", + build_args=("--select", "job:nightyl"), + expected_exit_code=1, + expected_tables=frozenset(), + expected_jobs_imported=True, + expected_fragment="unknown job name 'nightyl'", + ), + ], + ids=lambda case: case.description, +) +def test_given_declared_jobs_when_building_by_job_then_only_their_nodes_build( + test_case: JobSelectorBuildTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + for relative_path, contents in _PROJECT_FILES.items(): + destination: Path = tmp_path / relative_path + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(contents, encoding="utf-8") + + exit_code: int = main( + ["--project-dir", str(tmp_path), "--no-color", "build", "--no-tests", *test_case.build_args] + ) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == test_case.expected_exit_code, captured + assert test_case.expected_fragment in captured + assert (tmp_path / "jobs_imported").exists() == test_case.expected_jobs_imported + with duckdb.connect(str(tmp_path / "demo.duckdb")) as warehouse: + rows: list[tuple[str]] = warehouse.sql( + "SELECT table_name FROM duckdb_tables() " + "WHERE NOT internal AND NOT starts_with(table_name, '_sqlbuild')" + ).fetchall() + assert {name for (name,) in rows} == test_case.expected_tables diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 43ae2e1672..12c96c457a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -276,6 +276,8 @@ class SnapshotDiscoveryTestCase: expected_groups: dict[str, dict[str, list[str]]] = field(default_factory=dict) expected_sensors: dict[str, dict[str, object]] = field(default_factory=dict) expected_schedules: tuple[str, ...] = () + expected_declared_at: dict[str, dict[str, object] | None] = field(default_factory=dict) + expected_trigger_declared_at: dict[str, dict[str, object] | None] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index f7d26b84fa..c5325d6f57 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -192,6 +192,10 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr "fulfillment_calendar", "overdue_orders", } + assert first["documents"]["triggers"]["fulfillment_calendar"]["declared_at"] == { + "path": "orchestration/orders.py", + "line": 8, + } with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as control: stored, next_due = control.execute( "SELECT definition_json, next_due_at FROM sqb_triggers JOIN sqb_trigger_state USING (trigger_id) WHERE kind = 'schedule'" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py new file mode 100644 index 0000000000..79e8367e79 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py @@ -0,0 +1,98 @@ +"""A narrowed job run executes only the job's selected nodes and is recorded as a run of that job.""" + +import json +import sqlite3 +import subprocess +import sys +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import duckdb +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + WorkerExecutionTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + scheduled_worker, + wait_for_run, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="an operator excludes one model from a published job", + model_sql="SELECT 1 AS amount", + warehouse_query="SELECT table_name FROM duckdb_tables() ORDER BY table_name", + expected_status="success", + expected_rows=[], + files={ + "models/customers.sql": "MODEL (description 'Customers.', materialized table);\nSELECT 7 AS customer_id\n", + "models/products.sql": "MODEL (description 'Products.', materialized table);\nSELECT 3 AS product_id\n", + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job\n" + "nightly = job('nightly', description='Orders and customers.', select='orders customers')\n" + ), + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_published_job_when_an_operator_excludes_a_model_then_only_the_rest_of_the_job_runs( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + executable: str = str(Path(sys.executable).with_name("sqb")) + request_run: list[str] = [ + executable, + "admin", + "--database-env", + "SQLBUILD_SCHEDULER_TEST_STORE", + "request-run", + worker.api.location_id, + "nightly", + "--target", + "development", + "--reason", + "Rebuild the job without customers", + ] + refused: subprocess.CompletedProcess[str] = subprocess.run( + [*request_run, "--run-id", str(uuid4()), "--select", "products"], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert refused.returncode != 0 + assert "leave none of the job's nodes" in refused.stderr + run_id: str = str(uuid4()) + requested: subprocess.CompletedProcess[str] = subprocess.run( + [*request_run, "--run-id", run_id, "--exclude", "customers"], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert requested.returncode == 0, requested.stderr + assert json.loads(requested.stdout) == {"run_id": run_id, "applied": True} + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=run_id + ) + assert completed["status"] == test_case.expected_status, worker.log.read_text() + assert completed["job_key"] == "nightly" + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql( + "SELECT table_name FROM duckdb_tables() " + "WHERE NOT internal AND NOT starts_with(table_name, '_sqlbuild') ORDER BY 1" + ).fetchall() == [("orders",)] + with sqlite3.connect(worker.control_store) as control: + rows: list[tuple[str, str, str]] = control.execute( + "SELECT kind, selection_json, authored_json FROM sqb_runs" + ).fetchall() + assert len(rows) == 1 + kind, selection, authored = rows[0] + assert (kind, json.loads(selection)) == ("job", ["model:orders"]) + assert json.loads(authored)["request"]["exclude"] == ["customers"] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 9b1ba8d315..c6e520bb1f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -525,10 +525,14 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select "warehouse": "reporting", }, "concurrency_groups": ["order_reporting"], + "select": "+orders", + "exclude": "stg_orders", + "targets": ["development"], } } trigger: dict[str, Any] = first["documents"]["triggers"]["nightly_orders"] assert trigger["job_key"] == "nightly_orders" + assert trigger["targets"] == ["development"] assert trigger["schedule"] == { "kind": "cron", "expression": "15 2 * * *", @@ -746,6 +750,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 7, "default_status": "paused", + "declared_at": {"path": "orchestration/edge_cases/definitions.py", "line": 61}, }, "edge_job_outcomes": { "description": "Keep warning and partial outcomes distinct from success.", @@ -757,6 +762,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "paused", + "declared_at": {"path": "orchestration/edge_cases/definitions.py", "line": 77}, }, "edge_group_outcomes": { "description": "Observe group completion including cancellation and presumed failure.", @@ -769,9 +775,26 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "paused", + "declared_at": {"path": "orchestration/edge_cases/definitions.py", "line": 84}, }, }, expected_schedules=("edge_snapshot_orders",), + expected_trigger_declared_at={ + "edge_snapshot_orders": { + "path": "orchestration/edge_cases/definitions.py", + "line": 14, + }, + }, + expected_declared_at={ + "edge_snapshot_orders": { + "path": "orchestration/edge_cases/definitions.py", + "line": 14, + }, + "edge_snapshot_group": { + "path": "orchestration/edge_cases/definitions.py", + "line": 30, + }, + }, ), SnapshotDiscoveryTestCase( description="sensor-only roots collect source groups and destination jobs natively", @@ -793,6 +816,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject ), ), expected_jobs=("inventory", "source_orders"), + expected_declared_at={"inventory": None, "source_group": None}, expected_groups={"source_group": {"source_orders": []}}, expected_sensors={ "on_orders": { @@ -828,6 +852,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "running", + "declared_at": {"path": "orchestration/orders.py", "line": 7}, } }, ), @@ -852,6 +877,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "running", + "declared_at": {"path": "orchestration/orders.py", "line": 7}, } }, ), @@ -873,6 +899,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "when": {"kind": "source_new_data", "selection": ["source:raw_orders"]}, "minimum_interval_seconds": 1, "default_status": "running", + "declared_at": {"path": "orchestration/orders.py", "line": 10}, } }, ), @@ -898,6 +925,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 1, "default_status": "running", + "declared_at": {"path": "orchestration/orders.py", "line": 7}, } }, ), @@ -921,6 +949,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "running", + "declared_at": {"path": "orchestration/orders.py", "line": 7}, } }, ), @@ -939,6 +968,8 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, "minimum_interval_seconds": 7, "default_status": "paused", + "targets": ["development"], + "declared_at": {"path": "orchestration/health.py", "line": 9}, } }, ), @@ -982,6 +1013,17 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects ) for definition in publication["documents"].get("jobs", {}).values(): assert set(definition["selection"]) <= publication["nodes"].keys() + definitions: dict[str, Any] = { + **publication["documents"].get("jobs", {}), + **publication["documents"].get("groups", {}), + } + assert { + name: definitions[name].get("declared_at") for name in test_case.expected_declared_at + } == test_case.expected_declared_at + assert { + name: publication["documents"]["triggers"][name].get("declared_at") + for name in test_case.expected_trigger_declared_at + } == test_case.expected_trigger_declared_at @pytest.mark.parametrize( diff --git a/ui/src/lib/api/client.ts b/ui/src/lib/api/client.ts index ad23a18dbc..a7df56b9e7 100644 --- a/ui/src/lib/api/client.ts +++ b/ui/src/lib/api/client.ts @@ -1,5 +1,6 @@ import type { AdhocRunCheck, + PlanResult, AdhocRunRequest, Bootstrap, BulkRunResult, @@ -112,6 +113,8 @@ export interface ApiClient extends PlatformApi { /** The server's pre-launch check: resolution, write set and authorization. */ checkAdhocRun(scope: Scope, request: AdhocRunRequest): Promise; launchAdhocRun(scope: Scope, request: AdhocRunRequest): Promise; + /** Asks the location to run `sqb plan` on a build command against the warehouse, and waits for it. */ + planBuild(scope: Scope, command: string[]): Promise; listJobs(scope: Scope): Promise; getJob(scope: Scope, key: string): Promise; diff --git a/ui/src/lib/api/http-catalog.test.ts b/ui/src/lib/api/http-catalog.test.ts index 43a203dd0a..14d166ed95 100644 --- a/ui/src/lib/api/http-catalog.test.ts +++ b/ui/src/lib/api/http-catalog.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { components } from '$control/schema.d.ts'; -import { findNode, nodeDetailOf, nodeJobsOf, toCatalogNodes, toColumnLineage, toNodeRuns } from './http-catalog'; +import { findNode, nodeDetailOf, nodeJobsOf, toCatalogNodes, toColumnLineage, toMicrobatchCoverage, toNodeRuns } from './http-catalog'; import { toCheck } from './http-checks'; type W = components['schemas']; @@ -96,6 +96,41 @@ describe('catalog node detail', () => { const withSql = nodeDetailOf(nodes, nodes[0], null, { node_key: 'model:orders', snapshot_id: 'snap', raw_sql: 'select {{ x }}', compiled_sql: null }, []); expect(withSql.sql).toEqual({ raw: 'select {{ x }}', compiled: 'select {{ x }}' }); }); + + it('builds the warehouse object from the synced relation and its newest sample, and counts coverage holes', () => { + const [stored] = toCatalogNodes([node('model:orders', { materialization: 'table', relation: { schema: 'main', name: 'orders' } })]); + const sample = (observed_at: string, row_count: number | null, byte_count: number | null): W['WarehouseStatisticsRecord'] => ({ name: 'orders', state_schema: 'main', observed_at, row_count, byte_count }); + const statistics = [sample('2026-11-07T01:00:00Z', 10, 4096), sample('2026-11-07T02:00:00Z', null, null), sample('2026-11-07T03:00:00Z', 12, null)]; + const gap = { start: '20', end: '30', state: 'gap' as const }; + const coverage = toMicrobatchCoverage({ + model: 'orders', target_name: 'development', physical_generation_id: 'g', cursor_column: 'order_id', cursor_type: 'integer', batch_size: '10', + earliest: '0', contiguous_frontier: '20', known_gaps: [gap, { ...gap, start: '50', end: '60' }], batches_truncated: false, + batches: [{ start: '0', end: '10', state: 'complete', run_id: 'r1' }, { start: '10', end: '20', state: 'complete' }, gap] + }); + const baseTable: W['NodeStatus']['physical'] = { + status: 'observed', + observation: { event_id: 'o', observed_at: '2026-11-07T06:00:00Z', record: { name: 'orders', relation_type: 'BASE TABLE', state_schema: 'main', columns: [{ name: 'order_id', data_type: 'INTEGER' }] } } + }; + const d = nodeDetailOf([stored], stored, status({ status: 'matched', local_hash: 'a' }, baseTable), null, [], { statistics, microbatch: coverage }); + expect(d.warehouse).toEqual({ + relation: { database: null, schema: 'main', name: 'orders' }, + objectType: 'table', + createdAt: null, + lastAlteredAt: null, + rows: 12, + bytes: null, + columns: [{ name: 'order_id', declaredType: null, actualType: 'INTEGER', state: null }], + stats: [ + { at: '2026-11-07T01:00:00Z', rows: 10, bytes: 4096 }, + { at: '2026-11-07T03:00:00Z', rows: 12, bytes: null } + ], + syncedAt: '2026-11-07T06:00:00Z' + }); + expect(d.lastRows).toBe(12); + expect([d.microbatch?.knownGaps, d.microbatch?.batches.map((b) => b.runId)]).toEqual([2, ['r1', null, null]]); + const unsynced = nodeDetailOf([stored], stored, status({ status: 'matched', local_hash: 'a' }, { status: 'unavailable', reason: 'no catalog' }), null, [], { statistics }); + expect([unsynced.warehouse, unsynced.microbatch]).toEqual([null, null]); + }); }); describe('catalog node runs and jobs', () => { diff --git a/ui/src/lib/api/http-catalog.ts b/ui/src/lib/api/http-catalog.ts index 15c85d72c9..28f72202b5 100644 --- a/ui/src/lib/api/http-catalog.ts +++ b/ui/src/lib/api/http-catalog.ts @@ -1,7 +1,7 @@ import type { components } from '$control/schema.d.ts'; import { shortName } from '$lib/build-area/labels'; import { cardStatus } from '$lib/lineage/catalog-graph'; -import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, ColumnLens, ColumnLineage, NodeJob, NodeOutcome, NodeRunEntry, StaleReason } from './types/build'; +import type { CatalogNode, CatalogNodeDetail, Check, CheckCounts, ColumnLens, ColumnLineage, MicrobatchCoverage, NodeJob, NodeOutcome, NodeRunEntry, StaleReason, WarehouseObject } from './types/build'; import type { NodeKind } from './types/operate'; type Wire = components['schemas']; @@ -105,9 +105,47 @@ export function toNodeRuns(wire: Wire['NodeRunRecord'][]): NodeRunEntry[] { } /** - * A listed node with its status, SQL, run history, jobs and checks. The server keeps no row - * counts, cost or migrations per node, so those stay empty or null and the pages hide them. `sql` - * is null when the person may not see SQL; `runs` and `jobs` are null when their reads failed. + * The node's relation as last synced, with its sampled row counts. The sync records neither + * declared types nor creation times, so columns are not compared and those times stay null. + * Null until a sync has observed the relation. + */ +export function warehouseOf(node: CatalogNode, status: Wire['NodeStatus'] | null, statistics: Wire['WarehouseStatisticsRecord'][]): WarehouseObject | null { + if (!node.relation || status?.physical.status !== 'observed') return null; + const { observed_at, record } = status.physical.observation; + const stats = statistics.flatMap((r) => (r.row_count === null || r.row_count === undefined ? [] : [{ at: r.observed_at, rows: r.row_count, bytes: r.byte_count ?? null }])); + const newest = statistics.at(-1); + return { + relation: node.relation, + objectType: record.relation_type.toLowerCase().replace(/^base table$/, 'table'), + createdAt: null, + lastAlteredAt: null, + rows: newest?.row_count ?? null, + bytes: newest?.byte_count ?? null, + columns: record.columns.map((c) => ({ name: c.name, declaredType: null, actualType: c.data_type, state: null })), + stats, + syncedAt: observed_at + }; +} + +/** Recorded coverage of a concurrent microbatch model, with its holes counted. */ +export function toMicrobatchCoverage(wire: Wire['MicrobatchCoverage']): MicrobatchCoverage { + return { + cursorColumn: wire.cursor_column, + cursorType: wire.cursor_type, + batchSize: wire.batch_size, + earliest: wire.earliest, + contiguousFrontier: wire.contiguous_frontier ?? null, + knownGaps: wire.known_gaps.length, + batches: wire.batches.map((b) => ({ start: b.start, end: b.end, state: b.state, runId: b.run_id ?? null })), + batchesTruncated: wire.batches_truncated + }; +} + +/** + * A listed node with its status, SQL, run history, jobs, checks and synced warehouse object. The + * server keeps no cost or migrations per node, so those stay empty or null and the pages hide + * them. `sql` is null when the person may not see SQL; `runs` and `jobs` are null when their + * reads failed. */ export function nodeDetailOf( nodes: CatalogNode[], @@ -115,7 +153,7 @@ export function nodeDetailOf( status: Wire['NodeStatus'] | null, sql: Wire['NodeSql'] | null, checks: Check[], - { runs, jobs }: { runs: NodeRunEntry[] | null; jobs: NodeJob[] | null } = { runs: null, jobs: null } + { runs = null, jobs = null, statistics = [], microbatch = null }: { runs?: NodeRunEntry[] | null; jobs?: NodeJob[] | null; statistics?: Wire['WarehouseStatisticsRecord'][]; microbatch?: MicrobatchCoverage | null } = {} ): CatalogNodeDetail { // Checks name their node without its kind, so the attachment tells a source from a model of the same name. const own = checks.filter((c) => c.node === node.key || `${c.attachment}:${c.node}` === node.key); @@ -126,7 +164,7 @@ export function nodeDetailOf( checks: checkCountsOf(own), downstream: nodes.filter((n) => n.upstream.includes(node.key)).map((n) => n.key), recentOutcomes: (runs ?? []).slice(0, 40).map((r) => r.outcome).reverse(), - lastRows: null, + lastRows: statistics.at(-1)?.row_count ?? null, rowsDelta: null, lastDurationMs: runs?.find((r) => r.durationMs !== null)?.durationMs ?? null, cost: null, @@ -137,8 +175,8 @@ export function nodeDetailOf( jobs, runs, checkResults: own, - warehouse: null, - microbatch: null, + warehouse: warehouseOf(node, status, statistics), + microbatch, history: null }; } diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index 38e35f1815..b9979ba996 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -701,3 +701,128 @@ describe('node error text', () => { expect(paths.filter((p) => p.endsWith('/error'))).toHaveLength(0); }); }); + +describe('listPlatformEventHistory', () => { + const event = (storage_id: number, target: string) => ({ + storage_id, + event_id: `e-${storage_id}`, + occurred_at: '2026-11-09T09:00:00Z', + location_id: '0191f0aa-orders', + target, + event: { kind: 'location_broken', location_id: '0191f0aa-orders', target, last_seen_at: '2026-11-09T08:00:00Z' } + }); + + /** The newest scan holds only another target's event, and the next scan holds the one asked for. */ + function server() { + const scans: (string | null)[] = []; + vi.stubGlobal('document', { cookie: '' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/bootstrap')) return Response.json({ login_mode: 'password', access: { name: 'ada', principal_id: 'p', actions: [], locations: [] } }); + if (url.pathname.endsWith('/platform/problems')) return Response.json({ scheduler_leader: true, scheduler_lease_expires_at: null, broken_locations: [], overdue_ticks: [] }); + if (url.pathname.endsWith('/platform/events')) { + const before = url.searchParams.get('before'); + scans.push(before); + return Response.json(before === null ? { events: [event(9, 'preview')], before: 9 } : { events: [event(4, 'development')], before: null }); + } + return Response.json([]); + }) + ); + return { scans, client: createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }) }; + } + + it("keeps scanning past a scan without the location's events until the server says none are older", async () => { + const { scans, client } = server(); + const history = await client.listPlatformEventHistory({ locationId: '0191f0aa-orders:development' }); + expect(history.map((r) => r.id)).toEqual(['e-4']); + expect(scans).toEqual([null, '9']); + }); +}); + +describe('getCatalogNode statistics', () => { + const HOUR = 3_600_000; + const now = Date.parse('2026-11-09T12:00:00Z'); + /** One sample an hour for 1,500 hours up to now, served oldest first in pages of 100 after a cursor, as the server does. */ + const stored = Array.from({ length: 1500 }, (_, i) => ({ + event_id: `00000000-0000-0000-0000-${String(i + 1).padStart(12, '0')}`, + record: { name: 'orders', state_schema: 'main', observed_at: new Date(now - (1499 - i) * HOUR).toISOString(), row_count: i + 1 } + })); + + function server(samples: typeof stored) { + const reads: string[] = []; + vi.stubGlobal('document', { cookie: '' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/nodes')) + return Response.json([{ node_key: 'model:orders', kind: 'model', name: 'orders', materialization: 'table', tags: [], dependencies: [], snapshot_id: 's', relation: { schema: 'main', name: 'orders' } }]); + if (url.pathname.endsWith('/statistics')) { + const at = url.searchParams.get('after_observed_at') ?? ''; + const id = url.searchParams.get('after_event_id') ?? ''; + reads.push(`${url.searchParams.get('schema')}.${url.searchParams.get('name')}`); + const page = samples.filter((s) => s.record.observed_at > at || (s.record.observed_at === at && s.event_id > id)).slice(0, 100); + const last = page.at(-1); + return Response.json({ records: page, next_after: page.length === 100 && last ? { observed_at: last.record.observed_at, event_id: last.event_id } : null }); + } + if (url.pathname.endsWith('/node-status')) return Response.json(null); + return Response.json([]); + }) + ); + return { reads, client: createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }) }; + } + + beforeEach(() => vi.useFakeTimers({ now, toFake: ['Date'] })); + afterEach(() => vi.useRealTimers()); + + it('shows the newest sample on the first open, however much older history the server keeps', async () => { + const { reads, client } = server(stored); + expect((await client.getCatalogNode({ locationId: '0191f0aa-orders:development' }, 'orders'))?.lastRows).toBe(1500); + expect(reads.every((r) => r === 'main.orders')).toBe(true); + expect(reads.length).toBeLessThanOrEqual(11); + }); + + it('reads only the samples after the newest one it holds on a refresh, and keeps the earlier ones', async () => { + const samples = stored.slice(0, 1499); + const { reads, client } = server(samples); + const scope = { locationId: '0191f0aa-orders:development' }; + expect((await client.getCatalogNode(scope, 'orders'))?.lastRows).toBe(1499); + const first = reads.length; + samples.push(stored[1499]); + const refreshed = await client.getCatalogNode(scope, 'orders'); + expect([refreshed?.lastRows, refreshed?.warehouse, reads.length - first]).toEqual([1500, null, 1]); + }); +}); + +describe('listWarehouseConnections', () => { + const access = (target: string) => ({ location_id: '0191f0aa-orders', name: 'Orders', target, actions: [] }); + + /** Development's health is readable; preview's is refused, as for a partial viewer. */ + function server() { + const health: string[] = []; + vi.stubGlobal('document', { cookie: '' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/bootstrap')) return Response.json({ login_mode: 'password', access: { name: 'ada', principal_id: 'p', actions: [], locations: [access('development'), access('preview')] } }); + if (path.endsWith('/health')) { + health.push(path); + return path.includes('/targets/preview/') ? Response.json({ error: 'refused' }, { status: 403 }) : Response.json([]); + } + return Response.json([]); + }) + ); + return { health, client: createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }) }; + } + + it('asks for a refused target once per page load, since each refusal is audited', async () => { + const { health, client } = server(); + expect((await client.listWarehouseConnections()).map((c) => c.target)).toEqual(['development']); + expect((await client.listWarehouseConnections()).map((c) => c.target)).toEqual(['development']); + expect(health.filter((p) => p.includes('/preview/'))).toHaveLength(1); + expect(health.filter((p) => p.includes('/development/'))).toHaveLength(2); + }); +}); diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 35e0d1a326..8e82b6ad32 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -1,23 +1,26 @@ import createFetch from 'openapi-fetch'; +import { shortName } from '$lib/build-area/labels'; +import { isActiveRun } from '$lib/status'; import type { components, paths } from '$control/schema.d.ts'; import { ApiError, type ApiClient } from './client'; import { checkDetailOf, toCheck, toCheckEvidence, toCheckSql } from './http-checks'; -import { findNode, nodeDetailOf, toCatalogNodes, nodeJobsOf, toNodeRuns, toColumnLineage } from './http-catalog'; +import { findNode, nodeDetailOf, toCatalogNodes, nodeJobsOf, toMicrobatchCoverage, toNodeRuns, toColumnLineage } from './http-catalog'; import { auditMatches, toAuditEntry } from './http-audit'; import { toSnapshotChanges } from './http-changes'; -import { manualConnectionTest, toLocationDetail, toLocationSummary, warehouseSyncOf } from './http-locations'; -import { toAccount, toRoleAssignments } from './http-people'; +import { manualConnectionTest, toLocationDetail, toLocationSummary, toWarehouseConnection, warehouseSyncOf } from './http-locations'; +import { toAccount, toRoleAssignments, toServerSettings } from './http-people'; import { toMaintenanceHistory } from './http-maintenance'; import { overviewOf } from './http-overview'; -import { toPlatformEvents, toPlatformHealth } from './http-problems'; +import { toPlatformEventRecords, toPlatformEvents, toPlatformHealth } from './http-problems'; import { roleDetailOf, toRoleDefinition } from './http-roles'; import { jobDetailOf, runWindowOf, toJob, toTick, toTrigger, toTriggerStateEvent, triggerDetailOf } from './http-jobs'; -import { RUN_WINDOW, groupRunNodes, toAdhocCheck, runDetailOf, runRowPage, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs'; +import { timelineOf } from './http-timeline'; +import { RUN_WINDOW, groupRunNodes, parseRunCursor, planResultOf, queueEntriesOf, toAdhocCheck, runDetailOf, runRowPageOver, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs'; import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals'; -import type { AuthMode, Bootstrap, Capability, Location, PlatformEvent, Scope, WarehouseSync } from './types'; -import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, LocationSummary, PlatformHealth, MaintenanceHistory, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform'; +import type { AuthMode, Bootstrap, Capability, Location, NavCounts, PlatformEvent, Scope, WarehouseSync } from './types'; +import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, ServerSettings, WarehouseConnection, LocationSummary, PlatformEventQuery, PlatformEventRecord, PlatformHealth, MaintenanceHistory, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform'; import type { CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, Overview, SnapshotChange, ColumnLineage } from './types/build'; -import type { AdhocRunCheck, AdhocRunRequest, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunNodeError, RunOutputLine, RunStatement, StatementSql } from './types/operate'; +import type { AdhocRunCheck, AdhocRunRequest, BulkRunResult, PlanResult, QueueEntry, Timeline, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunNodeError, RunOutputLine, RunStatement, StatementSql } from './types/operate'; type Wire = components['schemas']; @@ -32,13 +35,18 @@ const SUPPORTED: ReadonlySet = new Set([ 'listRuns', 'getRun', 'getRunDetail', + 'listQueue', + 'navCounts', + 'timeline', 'listRunEvents', 'listRunStatements', 'getStatementSql', 'listRunOutput', 'getRunNodeError', 'cancelRun', + 'cancelMatchingRuns', 'rerunRun', + 'rerunSelectedNodes', 'listJobs', 'getJob', 'launchJob', @@ -66,25 +74,44 @@ const SUPPORTED: ReadonlySet = new Set([ 'listLocations', 'getLocation', 'listPlatformEvents', + 'listPlatformEventHistory', 'platformHealth', 'listRoles', 'getRole', 'getWarehouseSync', 'maintenanceHistory', 'testConnection', + 'listWarehouseConnections', + 'serverSettings', 'listSnapshotNodes', 'checkAdhocRun', - 'launchAdhocRun' + 'launchAdhocRun', + 'planBuild' ]); /** How often a connection test asks for the location's answer, and the longest it waits. */ const CONNECTION_TEST_POLL_MS = 1000; const CONNECTION_TEST_WAIT_MS = 60_000; +/** How often the plan page asks for the location's plan; the server expires a plan after ten minutes. */ +const PLAN_POLL_MS = 750; + /** The server's largest audit page, and how many the audit log reads before it stops looking. */ const AUDIT_PAGE = 100; const AUDIT_PAGES = 10; +/** The server's largest platform event scan, and how many event history reads before it stops looking. */ +const EVENT_PAGE = 500; +const EVENT_SCANS = 10; + +/** How many statistics pages of 100 a node read follows, and how many samples a relation keeps. */ +const STATISTICS_PAGES = 10; +const STATISTICS_KEPT = 1000; +/** How many hours back a first statistics read starts, at one sample an hour with a page to spare. */ +const STATISTICS_FIRST_HOURS = STATISTICS_PAGES * 100 - 100; +const HOUR_MS = 3_600_000; +const NIL_UUID = '00000000-0000-0000-0000-000000000000'; + /** A UI location is one server location and target; its id carries both. */ export function locationKey(locationId: string, target: string): string { return `${locationId}:${target}`; @@ -239,6 +266,39 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt const outputRefused = new Set(); // A node's error text needs view.evidence on that node alone. const errorRefused = new Set(); + // Targets whose health the person may not read; each refusal is audited, so Connections asks once per page load. + const healthRefused = new Set(); + // Statistics samples per relation, oldest first, with the cursor after the newest; a refresh reads only newer samples. + const samples = new Map(); + async function relationStatistics(scope: Scope, relation: { database: string | null; schema: string; name: string }): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const key = JSON.stringify([scope.locationId, relation.database, relation.schema, relation.name]); + const held = samples.get(key); + // The server pages oldest first from its retention cutoff and keeps up to one sample an hour, so a first read starts + // late enough that its pages reach the newest sample. + const since = { observed_at: new Date(Date.now() - STATISTICS_FIRST_HOURS * HOUR_MS).toISOString(), event_id: NIL_UUID }; + let after = held?.after ?? since; + const fresh: Wire['StatisticsObservation'][] = []; + for (let page = 0; page < STATISTICS_PAGES; page++) { + const { data, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/statistics', { + params: { + path: { location_id: locationId, target }, + query: { database: relation.database ?? undefined, schema: relation.schema, name: relation.name, after_observed_at: after?.observed_at, after_event_id: after?.event_id } + } + }); + if (response.status === 401) onUnauthorized(); + // Statistics need the same view as the node; a refusal leaves the samples out rather than failing the page. + if (!data) return held?.records ?? []; + fresh.push(...data.records); + const last = data.records.at(-1); + if (last) after = { observed_at: last.record.observed_at, event_id: last.event_id }; + if (!data.next_after) break; + } + if (!held && !fresh.length) return []; + const records = [...(held?.records ?? []), ...fresh.map((o) => o.record)].slice(-STATISTICS_KEPT); + samples.set(key, { records, after }); + return records; + } const implemented = { supports: (method: keyof ApiClient) => SUPPORTED.has(method), changes: new HttpChangeFeed(baseUrl, (scope) => feedAllowed(baseUrl, scope, onUnauthorized)), @@ -251,7 +311,21 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt evidenceLocations = new Set(bootstrap.locations.filter((l) => l.capabilities?.includes('view.evidence')).map((l) => l.id)); return bootstrap; }, - listRunRows: async (scope: Scope, query: RunRowQuery = {}): Promise => runRowPage(await listedRuns(scope), scope.locationId, query), + listRunRows: async (scope: Scope, query: RunRowQuery = {}): Promise => + runRowPageOver(await runWindows(scope, (w) => w.length > parseRunCursor(query.cursor).older), scope.locationId, query), + async cancelMatchingRuns(scope: Scope, query: RunRowQuery): Promise { + const { rows } = runRowPageOver([await listedRuns(scope)], scope.locationId, { ...query, cursor: null, limit: RUN_WINDOW }); + const result: BulkRunResult = { done: [], failed: [] }; + for (const { run } of rows.filter((row) => isActiveRun(row.run.status) && !row.cancelRequestedAt)) { + const body = { run_id: run.id, event_id: crypto.randomUUID(), reason: 'Cancelled from the web UI' }; + const { error, response } = await http.POST('/api/control/runs/cancel', { body, params: { header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); + if (response.status === 401) onUnauthorized(); + if (response.ok) result.done.push(run.id); + else result.failed.push({ runId: run.id, error: errorText(error, response) }); + } + forgetPending(); + return result; + }, runTabCounts: async (scope: Scope, query: RunRowQuery = {}): Promise => runTabCountsOf(await listedRuns(scope), scope.locationId, query), getRun: async (scope: Scope, runId: string): Promise => (await findRun(scope, runId))?.run ?? null, async getRunDetail(scope: Scope, runId: string): Promise { @@ -260,7 +334,30 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt const { run, wire, listed } = found; const owners = [run, ...listed.filter((r) => r.parentRunId === run.id)]; const [own, ...children] = await Promise.all(owners.map(async (owner) => (await runNodes(scope, owner.id)).map((n) => toRunNode(n, owner)))); - return runDetailOf(run, wire, listed, groupRunNodes(own, children.flat())); + const detail = runDetailOf(run, wire, listed, groupRunNodes(own, children.flat())); + if (run.status === 'queued') detail.waits = (await queueEntries(scope)).find((q) => q.run.id === run.id)?.waits ?? []; + return detail; + }, + listQueue: (scope: Scope): Promise => queueEntries(scope), + async navCounts(scope: Scope): Promise { + // The nodes listing goes first: a person who can't list the whole target gets one audited refusal, not five. + const nodes = await implemented.listCatalogNodes(scope); + const [checks, runs, triggers, events] = await Promise.all([implemented.listChecks(scope), listedRuns(scope), listedTriggers(scope), problems()]); + return { + catalogNodes: nodes.length, + failingChecks: checks.filter((c) => c.latest && ['fail', 'error', 'warn'].includes(c.latest.outcome)).length, + activeRuns: toRuns(runs, scope.locationId).filter((r) => !r.parentRunId && isActiveRun(r.status)).length, + pausedTriggers: triggers.filter((t) => t.status === 'paused').length, + activePlatformEvents: toPlatformEvents(events, Date.now()).filter((e) => e.active).length + }; + }, + async timeline(scope: Scope, window: { since: string; until: string }): Promise { + // Runs back to the window's start, at most ten windows; each schedule's recent ticks for covered markers. + const reached = (w: Wire['RunRecord'][][]) => w.length >= 10 || (w.at(-1)?.at(-1)?.queued_at ?? '') < window.since; + const [windows, jobs, triggers] = await Promise.all([runWindows(scope, reached), listedJobs(scope), listedTriggers(scope)]); + const schedules = triggers.filter((t) => 'schedule' in t.definition_json); + const ticks = (await Promise.all(schedules.map((t) => ticksOf(scope, t.trigger_id, 200)))).flat(); + return timelineOf({ runs: windows.flat(), jobs, triggers, ticks }, scope.locationId, window, Date.now()); }, async listRunEvents(scope: Scope, runId: string, query: { after?: number; limit?: number } = {}): Promise { const limit = query.limit ?? 1000; @@ -312,16 +409,25 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt async cancelRun(scope: Scope, runId: string): Promise { const body = { run_id: runId, event_id: crypto.randomUUID(), reason: 'Cancelled from the web UI' }; const { error, response } = await http.POST('/api/control/runs/cancel', { body, params: { header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); - inFlight.clear(); + forgetPending(); if (response.status === 401) onUnauthorized(); if (!response.ok) throw new ApiError(errorText(error, response), response.status); return requireRun(scope, runId); }, + async rerunSelectedNodes(scope: Scope, runId: string, nodes: string[]): Promise { + const source = await requireRun(scope, runId); + const body = { run_id: crypto.randomUUID(), source_run_id: runId, from_failure: false, nodes, priority: source.priority, reason: 'Rerun selected nodes from the web UI' }; + const { error, response } = await http.POST('/api/control/runs/rerun', { body, params: { header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); + forgetPending(); + if (response.status === 401) onUnauthorized(); + if (!response.ok) throw new ApiError(errorText(error, response), response.status); + return requireRun(scope, body.run_id); + }, async rerunRun(scope: Scope, runId: string, mode: Exclude): Promise { const source = await requireRun(scope, runId); const body = { run_id: crypto.randomUUID(), source_run_id: runId, from_failure: mode === 'from_failure', priority: source.priority, reason: 'Rerun from the web UI' }; const { error, response } = await http.POST('/api/control/runs/rerun', { body, params: { header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); - inFlight.clear(); + forgetPending(); if (response.status === 401) onUnauthorized(); if (!response.ok) throw new ApiError(errorText(error, response), response.status); return requireRun(scope, body.run_id); @@ -371,11 +477,36 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt const { locationId, target } = parseLocationKey(scope.locationId); const body = { run_id: crypto.randomUUID(), location_id: locationId, target, command: request.command, priority: request.priority, reason: 'Started from the web UI' }; const { error, response } = await http.POST('/api/control/runs/adhoc', { body, params: { header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); - inFlight.clear(); + forgetPending(); if (response.status === 401) onUnauthorized(); if (!response.ok) throw new ApiError(errorText(error, response), response.status); return requireRun(scope, body.run_id); }, + /** Queues the plan, then waits for the location's document; a plan the location could not make is a result, not a throw. */ + async planBuild(scope: Scope, command: string[]): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const path = { location_id: locationId, target }; + const requestId = crypto.randomUUID(); + const asked = await http.POST('/api/control/locations/{location_id}/targets/{target}/plans', { + body: { request_id: requestId, command }, + params: { path, header: { 'x-sqb-csrf': csrfToken(document.cookie) } } + }); + if (asked.response.status === 401) onUnauthorized(); + if (asked.response.status === 400) return { plan: null, error: errorText(asked.error, asked.response) }; + if (!asked.response.ok) throw new ApiError(errorText(asked.error, asked.response), asked.response.status); + for (;;) { + await new Promise((resolve) => setTimeout(resolve, PLAN_POLL_MS)); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/plans/{request_id}', { + params: { path: { ...path, request_id: requestId } } + }); + if (response.status === 401) onUnauthorized(); + if (!response.ok) throw new ApiError(errorText(error, response), response.status); + if (!data) throw new ApiError('The plan request disappeared', 404); + const outcome = data.result_json?.outcome; + if (outcome) return planResultOf(outcome); + if (Date.parse(data.expires_at) <= Date.now()) return { plan: null, error: 'The location did not plan the command in time; check that sqb location serve is running' }; + } + }, async listSnapshotChanges(scope: Scope, query: { limit?: number } = {}): Promise { const { locationId, target } = parseLocationKey(scope.locationId); const [snapshots, runs] = await Promise.all([ @@ -441,6 +572,31 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt return toMaintenanceHistory(data); }, listPlatformEvents: async (): Promise => toPlatformEvents(await problems(), Date.now()), + async listPlatformEventHistory(query: PlatformEventQuery = {}): Promise { + const limit = query.limit ?? 200; + const wanted = (wire: Wire['PersonPlatformEvent']) => + (query.locationId === undefined || (wire.location_id && wire.target ? locationKey(wire.location_id, wire.target) : null) === query.locationId) && + (!query.kinds?.length || query.kinds.includes(wire.event.kind)); + /** Each scan skips the events the person may not see, so one can come back short; keep scanning until enough match. */ + async function storedEvents(): Promise { + const kept: Wire['PersonPlatformEvent'][] = []; + let before: number | undefined; + for (let scan = 0; scan < EVENT_SCANS && kept.length < limit; scan++) { + const { data, error, response } = await http.GET('/api/control/platform/events', { params: { query: { limit: EVENT_PAGE, before } } }); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + kept.push(...data.events.filter(wanted)); + if (data.before === null || data.before === undefined) break; + before = data.before; + } + return kept.slice(0, limit); + } + const [open, stored, locations] = await Promise.all([problems().then((p) => toPlatformEvents(p, Date.now())), storedEvents(), bootstrapLocations()]); + const records = toPlatformEventRecords(open, stored, new Map(locations.map((l) => [l.id, l.name]))); + return records.filter( + (r) => (query.locationId === undefined || r.locationId === query.locationId) && (!query.kinds?.length || query.kinds.includes(r.kind)) && (!query.activeOnly || r.active) + ); + }, platformHealth: async (): Promise => toPlatformHealth(await problems()), /** Each location the person sees, with health only where they may list the whole target. */ async listLocations(): Promise { @@ -453,6 +609,20 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt }) ); }, + /** Targets whose health the person may not read are left out, as their connection checks are part of it. */ + async listWarehouseConnections(): Promise { + const locations = (await bootstrapLocations()).filter((l) => !healthRefused.has(l.id)); + const read = await Promise.all( + locations.map(async (location) => { + const { locationId, target } = parseLocationKey(location.id); + const { data, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/health', { params: { path: { location_id: locationId, target } } }); + if (response.status === 401) onUnauthorized(); + if (response.status === 403) healthRefused.add(location.id); + return data ? [toWarehouseConnection(location, data)] : []; + }) + ); + return read.flat(); + }, /** Null for a location the person can't see; a refused health read fails the page, as the target's listings do. */ async getLocation(locationId: string): Promise { const location = (await bootstrapLocations()).find((l) => l.id === locationId); @@ -462,6 +632,12 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt return toLocationDetail(location, health.data, snapshot, Date.now()); }, listAccounts: async (): Promise => (await people()).map(toAccount), + async serverSettings(): Promise { + const { data, error, response } = await http.GET('/api/admin/server'); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return toServerSettings(data); + }, listRoles: async (locationId: string): Promise => (await roleOverviews(locationId)).map(toRoleDefinition), async getRole(locationId: string, name: string): Promise { const [roles, assignments] = await Promise.all([roleOverviews(locationId), implemented.listRoleAssignments()]); @@ -508,7 +684,7 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt if (!node) return null; const { locationId, target } = parseLocationKey(scope.locationId); const path = { location_id: locationId, target }; - const [status, sql, checks, runs, jobs] = await Promise.all([ + const [status, sql, checks, runs, jobs, statistics, coverage] = await Promise.all([ http.GET('/api/control/locations/{location_id}/targets/{target}/node-status', { params: { path, query: { node_key: node.key } } }), sqlLocations.has(scope.locationId) && !sqlRefused.has(`${scope.locationId}|${node.key}`) ? http.GET('/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/sql', { params: { path: { ...path, node_key: node.key } } }) @@ -517,12 +693,21 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt node.kind === 'source' ? null : http.GET('/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/runs', { params: { path: { ...path, node_key: node.key }, query: { limit: 100 } } }), - Promise.all([listedJobs(scope), listedTriggers(scope)]).then(([j, t]) => nodeJobsOf(node.key, j, t)).catch(() => null) + Promise.all([listedJobs(scope), listedTriggers(scope)]).then(([j, t]) => nodeJobsOf(node.key, j, t)).catch(() => null), + node.relation ? relationStatistics(scope, node.relation) : [], + node.materialization?.includes('microbatch') + ? http.GET('/api/control/locations/{location_id}/targets/{target}/microbatches/{model}', { params: { path: { ...path, model: shortName(node.key) } } }) + : null ]); if (status.response.status === 401 || sql?.response.status === 401 || runs?.response.status === 401) onUnauthorized(); if (sql?.response.status === 403) sqlRefused.add(`${scope.locationId}|${node.key}`); // Status and SQL have their own permissions; a refusal leaves that part out rather than failing the page. - return nodeDetailOf(nodes, node, status.data ?? null, sql?.data ?? null, checks.map(toCheck), { runs: runs?.data ? toNodeRuns(runs.data) : null, jobs }); + return nodeDetailOf(nodes, node, status.data ?? null, sql?.data ?? null, checks.map(toCheck), { + runs: runs?.data ? toNodeRuns(runs.data) : null, + jobs, + statistics, + microbatch: coverage?.data ? toMicrobatchCoverage(coverage.data) : null + }); }, async overview(scope: Scope): Promise { // A week of publications for the models metric; the server returns at most 100. @@ -569,8 +754,12 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt const [wires, runs] = await Promise.all([listedTriggers(scope), recentRuns(scope)]); const wire = wires.find((t) => t.trigger_id === id); if (!wire) return null; - const [ticks, changes] = await Promise.all([ticksOf(scope, id, 100).then((t) => t.map(toTick)), triggerChangesOf(scope, id)]); - return triggerDetailOf(toTrigger(wire, runs, ticks[0] ?? null), wire, ticks, changes.map(toTriggerStateEvent)); + const [ticks, changes, sensor] = await Promise.all([ + ticksOf(scope, id, 100).then((t) => t.map(toTick)), + triggerChangesOf(scope, id), + 'when' in wire.definition_json ? sensorStateOf(scope, id) : null + ]); + return triggerDetailOf(toTrigger(wire, runs, ticks[0] ?? null), wire, ticks, changes.map(toTriggerStateEvent), sensor); }, async setTriggerStates(scope: Scope, ids: string[], state: TriggerState, reason?: string | null): Promise { for (const [done, id] of ids.entries()) { @@ -605,6 +794,12 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt * instead of each asking the server again. A write clears it, so reads after a write see it. */ const inFlight = new Map>(); + /** Windows of runs older than the newest, by the run each starts after; a mutation here forgets them. */ + const olderWindows = new Map>(); + function forgetPending() { + inFlight.clear(); + olderWindows.clear(); + } function shared(key: string, load: () => Promise): Promise { const pending = inFlight.get(key) as Promise | undefined; if (pending) return pending; @@ -654,6 +849,16 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt return data; } + async function sensorStateOf(scope: Scope, triggerId: string) { + const { locationId, target } = parseLocationKey(scope.locationId); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/triggers/{trigger_id}/sensor', { + params: { path: { location_id: locationId, target, trigger_id: triggerId }, query: { limit: 100 } } + }); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return data; + } + async function roleOverviews(locationKey: string) { const { locationId, target } = parseLocationKey(locationKey); const { data, error, response } = await http.GET('/api/admin/locations/{location_id}/targets/{target}/roles', { params: { path: { location_id: locationId, target } } }); @@ -757,13 +962,53 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt async function changeTrigger(triggerId: string, change: Wire['TriggerChange'], reason?: string | null) { const body = { trigger_id: triggerId, event_id: crypto.randomUUID(), actor: '', reason: reason?.trim() || 'Changed from the web UI', change }; const { error, response } = await http.POST('/api/control/triggers', { body, params: { header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); - inFlight.clear(); + forgetPending(); if (response.status === 401) onUnauthorized(); if (!response.ok) throw new ApiError(errorText(error, response), response.status); } const listedRuns = (scope: Scope) => shared(`runs ${scope.locationId}`, () => fetchRuns(scope)); + /** The newest window of runs, then older full windows until `enough` says so. */ + async function runWindows(scope: Scope, enough: (windows: Wire['RunRecord'][][]) => boolean) { + const windows = [await listedRuns(scope)]; + for (let last = windows[0]; !enough(windows) && last.length === RUN_WINDOW; windows.push(last)) { + last = await olderRuns(scope, last[last.length - 1]); + } + return windows; + } + + function olderRuns(scope: Scope, after: Wire['RunRecord']) { + const key = `${scope.locationId}|${after.queued_at}|${after.run_id}`; + const cached = olderWindows.get(key); + if (cached) return cached; + const { locationId, target } = parseLocationKey(scope.locationId); + const load = http + .GET('/api/control/locations/{location_id}/targets/{target}/runs', { + params: { path: { location_id: locationId, target }, query: { limit: RUN_WINDOW, before_queued_at: after.queued_at, before_run_id: after.run_id } } + }) + .then(({ data, error, response }) => { + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return data; + }); + olderWindows.set(key, load); + load.catch(() => olderWindows.delete(key)); + return load; + } + + /** Queued runs in claim order, with what each waits on. */ + const queueEntries = (scope: Scope) => + shared(`queue ${scope.locationId}`, async () => { + const { locationId, target } = parseLocationKey(scope.locationId); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/runs', { + params: { path: { location_id: locationId, target }, query: { queued: true, limit: RUN_WINDOW } } + }); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return queueEntriesOf(data, scope.locationId); + }); + /** The newest runs of the location, up to the server's cap. */ async function fetchRuns(scope: Scope) { const { locationId, target } = parseLocationKey(scope.locationId); @@ -786,10 +1031,22 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt async function startJob(scope: Scope, key: string, options: JobLaunchOptions | undefined, fullRefresh: boolean): Promise { const { locationId, target } = parseLocationKey(scope.locationId); const window = options?.window ? runWindowOf(options.window) : undefined; - const reason = fullRefresh ? 'Full refresh started from the web UI' : window ? 'Backfill started from the web UI' : 'Started from the web UI'; - const body = { run_id: crypto.randomUUID(), location_id: locationId, target, job_key: key, priority: 0, reason, window, full_refresh: fullRefresh || undefined }; + const narrowed = !!(options?.select?.length || options?.exclude?.length); + const reason = fullRefresh ? 'Full refresh started from the web UI' : window ? 'Backfill started from the web UI' : narrowed ? 'Part of the job started from the web UI' : 'Started from the web UI'; + const body = { + run_id: crypto.randomUUID(), + location_id: locationId, + target, + job_key: key, + priority: 0, + reason, + window, + full_refresh: fullRefresh || undefined, + select: options?.select?.length ? options.select : undefined, + exclude: options?.exclude?.length ? options.exclude : undefined + }; const { error, response } = await http.POST('/api/control/runs', { body, params: { header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); - inFlight.clear(); + forgetPending(); if (response.status === 401) onUnauthorized(); if (!response.ok) throw new ApiError(errorText(error, response), response.status); return requireRun(scope, body.run_id); diff --git a/ui/src/lib/api/http-jobs.test.ts b/ui/src/lib/api/http-jobs.test.ts index e158f11fbb..987a7ec3dc 100644 --- a/ui/src/lib/api/http-jobs.test.ts +++ b/ui/src/lib/api/http-jobs.test.ts @@ -7,8 +7,9 @@ type W = components['schemas']; const SNAPSHOT = '0191f0dd-0000-7000-8000-000000000000'; -function trigger(id: string, name: string, jobKey: string, definition: W['TriggerDeclaration'], status: W['TriggerStatus'] = 'running'): W['TriggerRecord'] { - return { trigger_id: id, name, job_key: jobKey, definition_json: definition, status, last_tick_for: null, next_due_at: '2026-11-07T06:00:00Z' }; +function trigger(id: string, name: string, jobKey: string, definition: W['TriggerDeclaration'], status: W['TriggerStatus'] = 'running'): W['PersonTriggerRecord'] { + const upcoming = 'schedule' in definition ? ['2026-11-07T06:00:00Z', '2026-11-08T06:00:00Z'] : []; + return { trigger_id: id, name, job_key: jobKey, definition_json: definition, status, last_tick_for: null, next_due_at: '2026-11-07T06:00:00Z', upcoming }; } const daily = trigger('t-daily', 'orders_daily', 'orders', { description: 'Daily orders', job_key: 'orders', schedule: { kind: 'cron', expression: '0 6 * * *', timezone: 'UTC' }, default_status: 'paused' }); @@ -40,6 +41,18 @@ describe('trigger mapping', () => { expect(toTrigger(onInventory, runs, null).lastRun).toBeNull(); }); + it('shows where a trigger is declared and the targets it is limited to, and nothing for an older snapshot', () => { + const declared = trigger('t-cal', 'fulfillment_calendar', 'orders', { + ...onInventory.definition_json, + targets: ['production'], + declared_at: { path: 'orchestration/orders.py', line: 8 } + } as W['SensorDefinition']); + const detail = triggerDetailOf(toTrigger(declared, [], null), declared, [], []); + expect([detail.sourceFile, detail.targets]).toEqual(['orchestration/orders.py:8', ['production']]); + const older = triggerDetailOf(toTrigger(daily, [], null), daily, [], []); + expect([older.sourceFile, older.targets]).toEqual([null, null]); + }); + it('links a group trigger to its group run, not a child listed before it', () => { const child = { ...run('r-child', 'orders_member', 't-daily'), parentRunId: 'r-group' } as Run; const group = { ...run('r-group', 'orders', 't-daily'), status: 'failed' } as Run; @@ -51,23 +64,68 @@ describe('trigger mapping', () => { expect(tick.lateBySeconds).toBe(150); const detail = triggerDetailOf(toTrigger(onInventory, [], tick), onInventory, [tick], []); expect(detail.sensor).toMatchObject({ minimumIntervalSeconds: 60, cursorKnown: false }); - expect([detail.declaration, detail.upcoming, detail.evaluations, detail.stateHistory]).toEqual([null, null, null, []]); - expect(triggerDetailOf(toTrigger(daily, [], null), daily, [], []).sensor).toBeNull(); + expect([detail.declaration, detail.upcoming, detail.catchUp, detail.evaluations, detail.stateHistory]).toEqual([null, null, null, null, []]); + const schedule = triggerDetailOf(toTrigger(daily, [], null), daily, [], []); + expect([schedule.sensor, schedule.upcoming, schedule.catchUp]).toEqual([null, ['2026-11-07T06:00:00Z', '2026-11-08T06:00:00Z'], { mode: 'latest', max: 1 }]); + }); + + it("shows a sensor's checks newest first, its cursor and when it moved, and flags code changed since the last check", () => { + const check = (id: string, at: string, before: W['PersonSensorCursor'], after: W['PersonSensorCursor'] | null, fingerprint: string): W['PersonSensorEvaluation'] => ({ + evaluation_id: id, + phase: after ? 'completed' : 'failed', + requested_at: at, + finished_at: at.replace(':00Z', ':05Z'), + cursor_before: before, + cursor_after: after, + run_count: after ? 2 : 0, + failure_code: after ? null : 'location_unreachable', + definition_fingerprint: fingerprint + }); + const state: W['PersonSensorState'] = { + cursor: { event_storage_id: 42, custom: 'feed/page-[REDACTED]' }, + definition_fingerprint: 'v2', + evaluations: [ + check('e-new', '2026-11-07T06:10:00Z', { event_storage_id: 40, custom: null }, null, 'v1'), + check('e-old', '2026-11-07T06:00:00Z', { event_storage_id: 0, custom: null }, { event_storage_id: 40, custom: null }, 'v1') + ], + run_requests: [{ run_key: 'stock-7', run_id: 'r-7', evaluation_id: 'e-old', queued_at: '2026-11-07T06:00:05Z', status: 'success' }] + }; + const reset = { id: 'c1', at: '2026-11-07T05:00:00Z', kind: 'cursor_reset', actor: { kind: 'user', name: 'ada', displayName: 'ada' }, reason: null, value: null } as const; + const detail = triggerDetailOf(toTrigger(onInventory, [], null), onInventory, [], [reset], state); + expect(detail.sensor).toMatchObject({ cursor: 'feed/page-[REDACTED]', cursorKnown: true, cursorUpdatedAt: '2026-11-07T06:00:05Z', currentFingerprint: 'v2', lastEvaluatedFingerprint: 'v1' }); + expect(detail.definitionChanged).toBe(true); + expect(detail.evaluations?.map((e) => [e.id, e.outcome, e.cursorBefore, e.cursorAfter, e.error])).toEqual([ + ['e-new', 'failed', 'event 40', null, 'location_unreachable'], + ['e-old', 'completed', null, 'event 40', null] + ]); + expect(detail.runRequests).toEqual([{ runKey: 'stock-7', runId: 'r-7', evaluationId: 'e-old', requestedAt: '2026-11-07T06:00:05Z', runStatus: 'success' }]); + const fresh = triggerDetailOf(toTrigger(onInventory, [], null), onInventory, [], [], { ...state, cursor: null, evaluations: [] }); + expect([fresh.sensor?.cursor, fresh.sensor?.cursorKnown, fresh.definitionChanged, fresh.sensor?.cursorUpdatedAt]).toEqual([null, true, false, null]); }); }); describe('job mapping', () => { - it('links a job to its triggers and newest run, and leaves unreported policy null', () => { - const wire: W['JobRecord'] = { job_key: 'orders', kind: 'job', description: 'Build orders', selection: ['model:customers', 'model:orders'], concurrency_groups: ['warehouse_heavy'], snapshot_id: SNAPSHOT }; + it('links a job to its triggers and newest run, shows where it is declared, and leaves unreported policy null', () => { + const wire: W['JobRecord'] = { + job_key: 'orders', + kind: 'job', + description: 'Build orders', + selection: ['model:customers', 'model:orders'], + concurrency_groups: ['warehouse_heavy'], + snapshot_id: SNAPSHOT, + select: '+orders', + declared_at: { path: 'orchestration/jobs.py', line: 12 } + }; const triggers = [toTrigger(daily, [], null), toTrigger(onInventory, [], null)]; const job = toJob(wire, triggers, [run('r-inv', 'inventory', null), run('r-ord', 'orders', null)]); - expect(job).toMatchObject({ key: 'orders', kind: 'job', nodeCount: 2, triggerIds: ['t-daily'], concurrencyGroups: ['warehouse_heavy'], select: null, overlap: null, priority: null }); + expect(job).toMatchObject({ key: 'orders', kind: 'job', nodeCount: 2, triggerIds: ['t-daily'], concurrencyGroups: ['warehouse_heavy'], select: '+orders', sourceFile: 'orchestration/jobs.py:12', overlap: null, priority: null }); expect(job.lastRun?.id).toBe('r-ord'); - const detail = jobDetailOf(job, { ...wire, settings: { vars: '[REDACTED]', concurrency: 4, empty_ok: true, models: ['a', 'b'] }, members: [], member_of: ['nightly'], critical_nodes: ['model:orders'] }); + const detail = jobDetailOf(job, { ...wire, settings: { vars: '[REDACTED]', concurrency: 4, empty_ok: true, models: ['a', 'b'] }, members: [], member_of: ['nightly'], critical_nodes: ['model:orders'], exclude: ' stg_orders audit_orders', targets: ['production'], cursor_type: 'integer' }); expect(detail.resolvedNodes).toEqual(['model:customers', 'model:orders']); expect(detail.settings).toEqual({ vars: '[REDACTED]', concurrency: '4', empty_ok: true, models: '["a","b"]' }); expect([detail.groupMembers, detail.members, detail.memberOf, detail.criticalNodes]).toEqual([null, null, ['nightly'], ['model:orders']]); - expect([detail.concurrency, detail.cursorType, detail.declaration]).toEqual([null, null, null]); + expect([detail.exclude, detail.targets, detail.cursorType]).toEqual([['stg_orders', 'audit_orders'], ['production'], 'integer']); + expect([detail.concurrency, detail.declaration]).toEqual([null, null]); expect(toJob({ ...wire, job_key: 'nightly', kind: 'group' }, triggers, []).kind).toBe('group'); }); @@ -82,6 +140,11 @@ describe('job mapping', () => { settings: {}, member_of: [], critical_nodes: [], + select: null, + exclude: null, + targets: [], + declared_at: null, + cursor_type: null, members: [ { job_key: 'stock', waits_for: [] }, { job_key: 'orders', waits_for: [] }, @@ -98,6 +161,7 @@ describe('job mapping', () => { ]); expect(detail.groupMembers?.flatMap((m) => m.upstream.map((u) => u.source))).toEqual([null, null, null]); expect(detail.members).toEqual(['stock', 'orders', 'customers', 'audit']); + expect([detail.select, detail.sourceFile, detail.exclude, detail.targets, detail.cursorType]).toEqual([null, null, [], null, null]); }); }); @@ -119,30 +183,18 @@ describe('runWindowOf', () => { }); describe('schedule due times', () => { - // Stored out of order on purpose; one time is before the server's next due time. + // Stored out of order on purpose. const times = ['2026-11-10T09:00:00Z', '2026-11-06T09:00:00Z', '2026-11-07T09:00:00Z', '2026-11-08T09:00:00Z']; - const calendar = (nextDueAt: string | null) => ({ - ...trigger('t-cal', 'business_days', 'orders', { description: 'Business days', job_key: 'orders', schedule: { kind: 'calendar', times, refresh_at: '2026-11-08T00:00:00Z' } }), - next_due_at: nextDueAt - }); - const detailOf = (wire: W['TriggerRecord']) => triggerDetailOf(toTrigger(wire, [], null), wire, [], []); - - it('lists a calendar from the next due time on, in time order, with the last stored time and next refresh', () => { - const detail = detailOf(calendar('2026-11-07T09:00:00Z')); - expect(detail.upcoming).toEqual(['2026-11-07T09:00:00Z', '2026-11-08T09:00:00Z', '2026-11-10T09:00:00Z']); - expect(detail.calendar).toMatchObject({ storedUntil: '2026-11-10T09:00:00.000Z', nextRefreshAt: '2026-11-08T00:00:00Z', horizon: null, low: false }); - }); - - it('has no upcoming calendar times once the server has no next due time', () => { - expect(detailOf(calendar(null)).upcoming).toEqual([]); - }); + const calendar = trigger('t-cal', 'business_days', 'orders', { description: 'Business days', job_key: 'orders', schedule: { kind: 'calendar', times, refresh_at: '2026-11-08T00:00:00Z' } }); + const detailOf = (wire: W['PersonTriggerRecord']) => triggerDetailOf(toTrigger(wire, [], null), wire, [], []); - it('shows only the next due time for cron, without working out later ones', () => { - expect(detailOf(daily).upcoming).toEqual(['2026-11-07T06:00:00Z']); + it('reports the last stored calendar time and the next refresh', () => { + expect(detailOf(calendar).calendar).toMatchObject({ storedUntil: '2026-11-10T09:00:00.000Z', nextRefreshAt: '2026-11-08T00:00:00Z', horizon: null, low: false }); expect(detailOf(daily).calendar).toBeNull(); }); - it('has no due times for a sensor', () => { + it('takes due times from the server, and has none for a sensor', () => { + expect(detailOf({ ...daily, upcoming: [] }).upcoming).toEqual([]); expect(detailOf(onInventory).upcoming).toBeNull(); }); }); diff --git a/ui/src/lib/api/http-jobs.ts b/ui/src/lib/api/http-jobs.ts index 02f7ccaf27..518072be7f 100644 --- a/ui/src/lib/api/http-jobs.ts +++ b/ui/src/lib/api/http-jobs.ts @@ -1,6 +1,7 @@ import type { components } from '$control/schema.d.ts'; import { ApiError } from './client'; -import type { CursorWindow, Job, JobDetail, JobGroupMember, Run, RunRef, Tick, Trigger, TriggerDetail, TriggerKind, TriggerStateEvent, TriggerStateEventKind } from './types/operate'; +import type { Iso } from './types'; +import type { CursorWindow, Job, JobDetail, JobGroupMember, Run, RunRef, SensorEvaluation, SensorEvaluationOutcome, Tick, Trigger, TriggerDetail, TriggerKind, TriggerStateEvent, TriggerStateEventKind } from './types/operate'; type Wire = components['schemas']; @@ -73,27 +74,12 @@ export function toTrigger(wire: Wire['TriggerRecord'], runs: Run[], lastTick: Ti lastTick, // Group children copy the group's trigger, so only top-level runs are the trigger's own. lastRun: newest(runs, (r) => r.triggerId === wire.trigger_id && !r.parentRunId), - sourceFile: null, + sourceFile: d.declared_at ? `${d.declared_at.path}:${d.declared_at.line}` : null, definitionChanged: false, warning: null }; } -/** How many upcoming due times a calendar schedule lists. */ -const UPCOMING_LIMIT = 5; - -/** - * The server's next due time, and for calendars the stored times from it on, which the scheduler - * takes in order. Later cron times would need cron evaluation, which stays on the server. - */ -function upcomingOf(d: Wire['TriggerDeclaration'], nextDueAt: string | null): string[] | null { - if (!isSchedule(d)) return null; - if (d.schedule.kind === 'cron') return nextDueAt ? [nextDueAt] : []; - if (!nextDueAt) return []; - const next = Date.parse(nextDueAt); - return d.schedule.times.filter((t) => Date.parse(t) >= next).sort((a, b) => Date.parse(a) - Date.parse(b)).slice(0, UPCOMING_LIMIT); -} - /** A trigger's detail; parts with no server source are null and their widgets stay hidden. */ const CHANGE_KINDS: Record = { enable: 'enabled', @@ -116,16 +102,53 @@ export function toTriggerStateEvent(wire: Wire['PersonTriggerChange']): TriggerS }; } -export function triggerDetailOf(trigger: Trigger, wire: Wire['TriggerRecord'], ticks: Tick[], history: TriggerStateEvent[]): TriggerDetail { +/** A built-in sensor's cursor is a control event position; a custom sensor's is its own value. */ +const shownCursor = (cursor: Wire['PersonSensorCursor']): string | null => cursor.custom ?? (cursor.event_storage_id > 0 ? `event ${cursor.event_storage_id}` : null); + +function toEvaluation(wire: Wire['PersonSensorEvaluation']): SensorEvaluation { + return { + id: wire.evaluation_id, + outcome: wire.phase as SensorEvaluationOutcome, + requestedAt: wire.requested_at, + finishedAt: wire.finished_at ?? null, + cursorBefore: shownCursor(wire.cursor_before), + cursorAfter: wire.cursor_after ? shownCursor(wire.cursor_after) : null, + runRequests: wire.run_count, + skipReason: null, + error: wire.failure_code ?? null, + definitionFingerprint: wire.definition_fingerprint + }; +} + +/** The cursor last moved when a check completed or an operator set or reset it, whichever is newer. */ +function cursorMovedAt(state: Wire['PersonSensorState'], history: TriggerStateEvent[]): Iso | null { + const times = [ + ...state.evaluations.filter((e) => e.phase === 'completed' && e.finished_at).map((e) => e.finished_at as string), + ...history.filter((h) => h.kind === 'cursor_set' || h.kind === 'cursor_reset').map((h) => h.at) + ]; + return times.length ? times.reduce((a, b) => (Date.parse(a) >= Date.parse(b) ? a : b)) : null; +} + +export function triggerDetailOf( + trigger: Trigger, + wire: Wire['PersonTriggerRecord'], + ticks: Tick[], + history: TriggerStateEvent[], + state: Wire['PersonSensorState'] | null = null +): TriggerDetail { const d = wire.definition_json; + const evaluations = state?.evaluations.map(toEvaluation) ?? null; + const lastEvaluatedFingerprint = state?.evaluations[0]?.definition_fingerprint ?? null; const calendar = isSchedule(d) && d.schedule.kind === 'calendar' ? d.schedule : null; const stored = calendar ? calendar.times.map(Date.parse) : []; return { ...trigger, + definitionChanged: !!state && lastEvaluatedFingerprint !== null && lastEvaluatedFingerprint !== state.definition_fingerprint, declaration: null, - targets: null, - catchUp: null, - upcoming: upcomingOf(d, wire.next_due_at ?? null), + targets: d.targets?.length ? d.targets : null, + // The scheduler ticks only the latest missed time after downtime, and a run still waiting covers it. + catchUp: isSchedule(d) ? { mode: 'latest', max: 1 } : null, + upcoming: isSchedule(d) ? wire.upcoming : null, calendar: calendar && { horizon: null, storedUntil: stored.length ? new Date(Math.max(...stored)).toISOString() : null, @@ -141,15 +164,16 @@ export function triggerDetailOf(trigger: Trigger, wire: Wire['TriggerRecord'], t nodeConditions: [], minimumIntervalSeconds: d.minimum_interval_seconds, timeoutSeconds: null, - cursor: null, - cursorUpdatedAt: null, - cursorKnown: false, - currentFingerprint: null, - lastEvaluatedFingerprint: null + cursor: state?.cursor ? shownCursor(state.cursor) : null, + cursorUpdatedAt: state ? cursorMovedAt(state, history) : null, + cursorKnown: !!state, + currentFingerprint: state?.definition_fingerprint ?? null, + lastEvaluatedFingerprint }, ticks, - evaluations: null, - runRequests: null, + evaluations, + runRequests: + state?.run_requests.map((r) => ({ runKey: r.run_key, runId: r.run_id, evaluationId: r.evaluation_id, requestedAt: r.queued_at, runStatus: r.status })) ?? null, stateHistory: history }; } @@ -159,8 +183,7 @@ export function toJob(wire: Wire['JobRecord'], triggers: Trigger[], runs: Run[]) key: wire.job_key, kind: wire.kind === 'group' ? 'group' : 'job', description: wire.description, - // The server stores the resolved node list, not the selector text. - select: null, + select: wire.select ?? null, members: null, nodeCount: wire.selection.length, overlap: null, @@ -168,7 +191,7 @@ export function toJob(wire: Wire['JobRecord'], triggers: Trigger[], runs: Run[]) concurrencyGroups: wire.concurrency_groups, priority: null, owner: null, - sourceFile: null, + sourceFile: wire.declared_at ? `${wire.declared_at.path}:${wire.declared_at.line}` : null, triggerIds: triggers.filter((t) => t.jobKey === wire.job_key).map((t) => t.id), lastRun: newest(runs, (r) => r.jobKey === wire.job_key) }; @@ -192,16 +215,17 @@ export function jobDetailOf(job: Job, wire: Wire['JobDetailRecord']): JobDetail return { ...job, members: wire.kind === 'group' ? wire.members.map((m) => m.job_key) : null, + // Published snapshots record where a job is declared but not its code. declaration: null, - exclude: [], + exclude: wire.exclude?.split(/\s+/).filter(Boolean) ?? [], settings: Object.fromEntries(Object.entries(wire.settings).map(([name, value]) => [name, shownSetting(value)])), resolvedNodes: wire.selection, concurrency: null, - targets: null, + targets: wire.targets.length ? wire.targets : null, groupMembers: wire.kind === 'group' ? groupMembersOf(wire.members) : null, criticalNodes: wire.critical_nodes, memberOf: wire.member_of, - cursorType: null + cursorType: wire.cursor_type === 'integer' || wire.cursor_type === 'timestamp' ? wire.cursor_type : null }; } diff --git a/ui/src/lib/api/http-locations.test.ts b/ui/src/lib/api/http-locations.test.ts index 5547d25018..5521f5162f 100644 --- a/ui/src/lib/api/http-locations.test.ts +++ b/ui/src/lib/api/http-locations.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { connectionTestOf, healthOf, instancesOf, toLocationDetail, warehouseSyncOf } from './http-locations'; +import { connectionTestOf, healthOf, instancesOf, toLocationDetail, toWarehouseConnection, warehouseSyncOf } from './http-locations'; const now = Date.parse('2026-11-08T12:00:00Z'); const at = (secondsAgo: number) => new Date(now - secondsAgo * 1000).toISOString(); @@ -76,6 +76,21 @@ describe('warehouseSyncOf', () => { expect(result).toEqual({ lastSyncedAt: '2026-11-08T12:02:00Z', lastError: 'catalog: failed while reading the warehouse' }); }); + it("gives each table its newest sync, the time it last synced cleanly, and where the state tables are read from", () => { + const location = { id: 'loc:development', name: 'Orders', health: null, lastSeenAt: null, target: { name: 'development', adapter: null, connectionSummary: null }, warehouseSync: { lastSyncedAt: null, lastError: null } }; + const connection = toWarehouseConnection(location, [ + sync('2026-11-08T12:03:00Z', 'statistics', 'read'), + sync('2026-11-08T12:02:00Z', 'catalog', null), + sync('2026-11-08T12:01:00Z', 'statistics', null), + sync('2026-11-08T12:00:00Z', 'statistics', null) + ]); + expect([connection.id, connection.target, connection.stateLocation, connection.lastTest]).toEqual(['loc:development', 'development', { database: null, schema: 'main' }, null]); + expect(connection.tables.map((t) => [t.table, t.at, t.failure, t.lastSyncedAt])).toEqual([ + ['catalog', '2026-11-08T12:02:00Z', null, '2026-11-08T12:02:00Z'], + ['statistics', '2026-11-08T12:03:00Z', 'Failed while reading the warehouse', '2026-11-08T12:01:00Z'] + ]); + }); + it('clears the error once the failing table syncs again', () => { expect(warehouseSyncOf([sync('2026-11-08T12:05:00Z', 'catalog', null), sync('2026-11-08T12:03:00Z', 'catalog', 'read')])?.lastError).toBeNull(); }); diff --git a/ui/src/lib/api/http-locations.ts b/ui/src/lib/api/http-locations.ts index b10cc818d8..32a21cd1c4 100644 --- a/ui/src/lib/api/http-locations.ts +++ b/ui/src/lib/api/http-locations.ts @@ -1,6 +1,6 @@ import type { components } from '$control/schema.d.ts'; import type { Location, LocationHealth, Snapshot, WarehouseSync } from './types'; -import type { ConnectionTest, LocationDetail, LocationInstance, LocationSummary } from './types/platform'; +import type { ConnectionTest, LocationDetail, LocationInstance, LocationSummary, WarehouseConnection, WarehouseTableSync } from './types/platform'; type Wire = components['schemas']; @@ -126,13 +126,44 @@ const SYNC_STAGE: Record = { delivery: 'sending results to the server' }; -/** When a table last synced cleanly, and which tables' newest sync failed; null without any sync report. */ -export function warehouseSyncOf(observations: Wire['LocationHealthObservation'][]): WarehouseSync | null { - type Sync = Wire['LocationHealthObservation'] & { report: { health: Extract } }; +type Sync = Wire['LocationHealthObservation'] & { report: { health: Extract } }; + +/** Sync reports newest first, and the newest of each table. */ +function syncsOf(observations: Wire['LocationHealthObservation'][]): { syncs: Sync[]; newest: Map } { const syncs = observations.filter((o): o is Sync => o.report.health.kind === 'warehouse_sync').sort((a, b) => b.received_at.localeCompare(a.received_at)); - if (!syncs.length) return null; const newest = new Map(); for (const sync of syncs) if (!newest.has(sync.report.health.table)) newest.set(sync.report.health.table, sync); + return { syncs, newest }; +} + +/** A location target's connection from its health reports: the newest connection check, where its state tables are read from, and each table's newest sync. */ +export function toWarehouseConnection(location: Location, observations: Wire['LocationHealthObservation'][]): WarehouseConnection { + const { syncs, newest } = syncsOf(observations); + const tables: WarehouseTableSync[] = [...newest.values()] + .map(({ received_at, report: { health } }) => ({ + table: health.table, + at: received_at, + recordsSent: health.records_sent, + elapsedMs: health.elapsed_millis, + failure: health.failure ? `Failed while ${SYNC_STAGE[health.failure]}` : null, + lastSyncedAt: syncs.find((s) => s.report.health.table === health.table && !s.report.health.failure)?.received_at ?? null + })) + .sort((a, b) => a.table.localeCompare(b.table)); + const state = syncs[0]?.report.health.location; + return { + id: location.id, + location: location.name, + target: location.target.name, + stateLocation: state ? { database: state.database ?? null, schema: state.schema } : null, + lastTest: connectionTestOf(observations), + tables + }; +} + +/** When a table last synced cleanly, and which tables' newest sync failed; null without any sync report. */ +export function warehouseSyncOf(observations: Wire['LocationHealthObservation'][]): WarehouseSync | null { + const { syncs, newest } = syncsOf(observations); + if (!syncs.length) return null; const failing = [...newest.values()].flatMap((s) => (s.report.health.failure ? [`${s.report.health.table}: failed while ${SYNC_STAGE[s.report.health.failure]}`] : [])); return { lastSyncedAt: syncs.find((s) => !s.report.health.failure)?.received_at ?? null, diff --git a/ui/src/lib/api/http-people.test.ts b/ui/src/lib/api/http-people.test.ts index 518daeb696..c70d6ed5fc 100644 --- a/ui/src/lib/api/http-people.test.ts +++ b/ui/src/lib/api/http-people.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { toAccount, toRoleAssignments } from './http-people'; +import { toAccount, toRoleAssignments, toServerSettings } from './http-people'; const person = (name: string, groups: string[], has_password = false) => ({ principal_id: `p-${name}`, @@ -36,3 +36,25 @@ describe('people', () => { ]); }); }); + +describe('toServerSettings', () => { + it('names each kept record with its own retention', () => { + const settings = toServerSettings({ + version: '0.205.1', + login_mode: 'password', + scheduler: false, + installation: { storage: 'sqlite', installed_at: '2026-11-01T00:00:00Z', schema_checksum: 'abc', schema_applied_at: '2026-11-01T00:00:00Z' }, + retention: { audit_days: 400, snapshots_kept: 10, statistics_days: 90, platform_alert_days: 30, location_health_days: 7, sensor_evaluation_days: 6, heartbeat_days: 1 } + }); + expect([settings.authMode, settings.schedulerEnabled, settings.storage]).toEqual(['password', false, 'sqlite']); + expect(settings.retention.map((r) => [r.name, r.value])).toEqual([ + ['Security audit log', '400 days'], + ['Published versions', '10'], + ['Warehouse row counts', '90 days'], + ['Platform problems', '30 days'], + ['Location check-ins', '7 days'], + ['Sensor checks', '6 days'], + ['Server heartbeats', '1 day'] + ]); + }); +}); diff --git a/ui/src/lib/api/http-people.ts b/ui/src/lib/api/http-people.ts index 9f3af49421..5ce06fac92 100644 --- a/ui/src/lib/api/http-people.ts +++ b/ui/src/lib/api/http-people.ts @@ -1,5 +1,5 @@ import type { components } from '$control/schema.d.ts'; -import type { Account, RoleAssignment } from './types/platform'; +import type { Account, RoleAssignment, ServerSettings } from './types/platform'; type Wire = components['schemas']; @@ -56,3 +56,28 @@ export function toRoleAssignments(holdings: Wire['RoleHoldings'], people: Wire[' ); return [...manual, ...viaGroups]; } + +const days = (n: number) => `${n} ${n === 1 ? 'day' : 'days'}`; + +/** The server's own facts, with each kind of record and how long the server keeps it. */ +export function toServerSettings(wire: Wire['ServerSettings']): ServerSettings { + const r = wire.retention; + return { + serverVersion: wire.version, + storage: wire.installation.storage, + installedAt: wire.installation.installed_at, + schemaChecksum: wire.installation.schema_checksum, + schemaAppliedAt: wire.installation.schema_applied_at, + authMode: wire.login_mode, + schedulerEnabled: wire.scheduler, + retention: [ + { name: 'Security audit log', value: days(r.audit_days), description: 'Who did what, and every refusal.' }, + { name: 'Published versions', value: String(r.snapshots_kept), description: 'Kept for each location target; older versions are removed.' }, + { name: 'Warehouse row counts', value: days(r.statistics_days), description: 'Row counts and sizes sampled from the warehouse.' }, + { name: 'Platform problems', value: days(r.platform_alert_days), description: 'Late schedules, failed launches and locations that stopped checking in.' }, + { name: 'Location check-ins', value: days(r.location_health_days), description: 'Health reports from location processes.' }, + { name: 'Sensor checks', value: days(r.sensor_evaluation_days), description: 'Each sensor check. Where each sensor is up to is always kept.' }, + { name: 'Server heartbeats', value: days(r.heartbeat_days), description: 'Check-ins from the servers themselves.' } + ] + }; +} diff --git a/ui/src/lib/api/http-problems.test.ts b/ui/src/lib/api/http-problems.test.ts index 553fa62ed0..523adcf75d 100644 --- a/ui/src/lib/api/http-problems.test.ts +++ b/ui/src/lib/api/http-problems.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { toPlatformEvents, toPlatformHealth } from './http-problems'; +import { toPlatformEventRecords, toPlatformEvents, toPlatformHealth } from './http-problems'; const NOW = Date.parse('2026-11-08T12:00:00Z'); const tick = (name: string, location: string, scheduledFor: string) => ({ @@ -31,6 +31,31 @@ describe('toPlatformEvents', () => { expect(broken).toMatchObject({ kind: 'location_broken', locationId: 'l2:development', occurredAt: '2026-11-08T11:55:00Z', subject: 'inventory · development' }); }); + it('groups low calendars per location, dated and named by the one that runs out first', () => { + const calendar = (name: string, location: string, horizon_end: string | null) => ({ + trigger_id: `c-${name}`, + name, + location_id: location, + location_name: location === 'l1' ? 'orders' : 'inventory', + target: 'development', + refresh_at: '2026-11-08T13:00:00Z', + horizon_end + }); + const events = toPlatformEvents( + { + scheduler_leader: true, + broken_locations: [], + overdue_ticks: [], + low_calendars: [calendar('month_end', 'l1', '2026-11-09T06:00:00Z'), calendar('quarter_end', 'l1', '2026-11-08T18:00:00Z'), calendar('stock_counts', 'l2', null)] + }, + NOW + ); + expect(events.map((e) => [e.kind, e.locationId, e.occurredAt, e.message])).toEqual([ + ['calendar_horizon_low', 'l1:development', '2026-11-08T18:00:00Z', '2 calendars in orders · development are running out of dates, starting with quarter_end.'], + ['calendar_horizon_low', 'l2:development', '2026-11-08T13:00:00Z', 'stock_counts in inventory · development has no dates left; the location works out more at its next refresh.'] + ]); + }); + it('groups late ticks per location, dated and named by the oldest', () => { const events = toPlatformEvents( { @@ -58,3 +83,31 @@ describe('toPlatformHealth', () => { expect(toPlatformHealth({ scheduler_leader: false, scheduler_lease_expires_at: '2026-11-08T11:50:00Z', broken_locations: [], overdue_ticks: [] }).scheduler.lease).toBeNull(); }); }); + +describe('toPlatformEventRecords', () => { + it('marks the stored event of an open problem as ongoing instead of listing it twice, and keeps each location target', () => { + const open = toPlatformEvents({ scheduler_leader: false, scheduler_lease_expires_at: null, broken_locations: [], overdue_ticks: [] }, NOW); + const records = toPlatformEventRecords( + open, + [ + { storage_id: 9, event_id: 'e9', occurred_at: '2026-11-08T11:00:00Z', location_id: 'l1', target: 'development', event: { kind: 'launch_failed', run_id: 'r1', claim_id: 'c1', code: 'image_missing' } }, + { storage_id: 4, event_id: 'e4', occurred_at: '2026-11-08T10:00:00Z', event: { kind: 'scheduler_no_leader' } } + ], + new Map([['l1:development', 'orders']]) + ); + expect(records.map((r) => [r.kind, r.active, r.locationId])).toEqual([ + ['launch_failed', false, 'l1:development'], + ['scheduler_no_leader', true, null] + ]); + expect(records[0]).toMatchObject({ subject: 'orders · development', message: "A run in orders · development couldn't start (image_missing).", links: [{ kind: 'run', id: 'r1', locationId: 'l1:development' }], payload: { code: 'image_missing' } }); + }); + + it('lists an open problem nothing was stored for first', () => { + const open = toPlatformEvents({ scheduler_leader: true, scheduler_lease_expires_at: null, broken_locations: [{ location_id: 'l2', location_name: 'inventory', target: 'development', last_seen_at: '2026-11-08T11:00:00Z' }], overdue_ticks: [] }, NOW); + const records = toPlatformEventRecords(open, [{ storage_id: 1, event_id: 'e1', occurred_at: '2026-11-08T09:00:00Z', location_id: 'l1', target: 'development', event: { kind: 'location_broken', location_id: 'l1', target: 'development', last_seen_at: '2026-11-08T08:59:00Z' } }], new Map()); + expect(records.map((r) => [r.kind, r.active, r.locationId])).toEqual([ + ['location_broken', true, 'l2:development'], + ['location_broken', false, 'l1:development'] + ]); + }); +}); diff --git a/ui/src/lib/api/http-problems.ts b/ui/src/lib/api/http-problems.ts index 53c466a61d..456997bde2 100644 --- a/ui/src/lib/api/http-problems.ts +++ b/ui/src/lib/api/http-problems.ts @@ -1,13 +1,13 @@ import type { components } from '$control/schema.d.ts'; import { locationKey as keyOf } from './http-client'; import type { PlatformEvent } from './types'; -import type { PlatformHealth } from './types/platform'; +import type { PlatformEventRecord, PlatformHealth } from './types/platform'; type Wire = components['schemas']; const placeOf = (name: string, target: string) => `${name} · ${target}`; -/** Live problems as active banner events: a stopped scheduler, each broken location, and late schedules per location. */ +/** Live problems as active banner events: a stopped scheduler, each broken location, and late schedules and low calendars per location. */ export function toPlatformEvents(problems: Wire['PlatformProblems'], now: number): PlatformEvent[] { const events: PlatformEvent[] = []; if (!problems.scheduler_leader) { @@ -53,6 +53,30 @@ export function toPlatformEvents(problems: Wire['PlatformProblems'], now: number message: ticks.length === 1 ? `${oldest.name} in ${place} is late.` : `${ticks.length} schedules in ${place} are late, starting with ${oldest.name}.` }); } + const low = new Map(); + for (const calendar of problems.low_calendars ?? []) { + const key = keyOf(calendar.location_id, calendar.target); + low.set(key, [...(low.get(key) ?? []), calendar]); + } + for (const [key, calendars] of low) { + const first = calendars[0]; + const place = placeOf(first.location_name, first.target); + const ends = (c: Wire['LowCalendar']) => c.horizon_end ?? c.refresh_at; + const soonest = calendars.reduce((a, b) => (ends(a) <= ends(b) ? a : b)); + events.push({ + id: `calendar:${key}`, + kind: 'calendar_horizon_low', + severity: 'warning', + occurredAt: ends(soonest), + active: true, + locationId: key, + subject: place, + message: + calendars.length === 1 + ? `${soonest.name} in ${place} ${soonest.horizon_end ? 'is running out of dates' : 'has no dates left'}; the location works out more at its next refresh.` + : `${calendars.length} calendars in ${place} are running out of dates, starting with ${soonest.name}.` + }); + } return events; } @@ -64,3 +88,36 @@ export function toPlatformHealth(problems: Wire['PlatformProblems']): PlatformHe heartbeat: null }; } + +/** Stored events newest first; an open problem marks the newest stored event of its kind and place as ongoing, or comes first when none was stored. */ +export function toPlatformEventRecords(open: PlatformEvent[], stored: Wire['PersonPlatformEvent'][], names: Map): PlatformEventRecord[] { + const records = stored.map((wire) => storedRecord(wire, names)); + const unmatched: PlatformEventRecord[] = []; + for (const event of open) { + const latest = records.find((r) => !r.active && r.kind === event.kind && r.locationId === event.locationId); + if (latest) latest.active = true; + else unmatched.push({ ...event, resolvedAt: null, links: [], reactions: [], payload: {} }); + } + return [...unmatched, ...records]; +} + +function storedRecord(wire: Wire['PersonPlatformEvent'], names: Map): PlatformEventRecord { + const key = wire.location_id && wire.target ? keyOf(wire.location_id, wire.target) : null; + const place = key && wire.target ? placeOf(names.get(key) ?? 'A location', wire.target) : null; + const base = { id: wire.event_id, occurredAt: wire.occurred_at, active: false, locationId: key, subject: place, resolvedAt: null, reactions: [] }; + const event = wire.event; + switch (event.kind) { + case 'scheduler_no_leader': + return { ...base, kind: event.kind, severity: 'critical', message: 'No server held the scheduler lease, so schedules and sensors stopped starting runs.', links: [], payload: {} }; + case 'location_broken': + return { ...base, kind: event.kind, severity: 'critical', message: `${place} stopped checking in.`, links: key ? [{ kind: 'location', id: key }] : [], payload: { last_seen_at: event.last_seen_at } }; + case 'tick_overdue': + return { ...base, kind: event.kind, severity: 'warning', message: `A schedule in ${place ?? 'a location'} was late.`, links: key ? [{ kind: 'trigger', id: event.trigger_id, name: 'schedule', locationId: key }] : [], payload: { scheduled_for: event.scheduled_for } }; + case 'calendar_horizon_low': + return { ...base, kind: event.kind, severity: 'warning', message: `A calendar in ${place ?? 'a location'} was running out of dates.`, links: key ? [{ kind: 'trigger', id: event.trigger_id, name: 'calendar', locationId: key }] : [], payload: { refresh_at: event.refresh_at, ...(event.horizon_end ? { horizon_end: event.horizon_end } : {}) } }; + case 'launch_failed': + return { ...base, kind: event.kind, severity: 'critical', message: `A run in ${place ?? 'a location'} couldn't start (${event.code}).`, links: key ? [{ kind: 'run', id: event.run_id, locationId: key }] : [], payload: { code: event.code } }; + case 'heartbeat': + return { ...base, kind: event.kind, severity: 'info', message: 'A server checked in.', links: [], payload: {} }; + } +} diff --git a/ui/src/lib/api/http-runs.test.ts b/ui/src/lib/api/http-runs.test.ts index 97ec36118b..fbc616e519 100644 --- a/ui/src/lib/api/http-runs.test.ts +++ b/ui/src/lib/api/http-runs.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { components } from '$control/schema.d.ts'; -import { groupRunNodes, runDetailOf, runRowPage, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunStatements, toRunOutput } from './http-runs'; +import { groupRunNodes, parseRunCursor, queueEntriesOf, planResultOf, runDetailOf, runRowPage, runRowPageOver, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunStatements, toRunOutput } from './http-runs'; type RunRecord = components['schemas']['RunRecord']; type W = components['schemas']; @@ -46,6 +46,11 @@ describe('run mapping', () => { expect(scheduler.requestedBy).toEqual({ kind: 'scheduler', name: 'system:scheduler', displayName: 'system:scheduler' }); }); + it('keeps how a run of part of a job was narrowed, and nothing for a whole-job run', () => { + const [part, whole] = toRuns([wire('p', { narrowing: { select: [], exclude: ['customers'] } }), wire('w')], LOCATION); + expect([part.narrowing, whole.narrowing]).toEqual([{ select: [], exclude: ['customers'] }, null]); + }); + it('counts cancelled nodes as failed and counts each run’s direct children', () => { const runs = toRuns( [ @@ -100,6 +105,95 @@ describe('runs list', () => { }); }); +describe('backfill window', () => { + it('shows both ends of an integer or timestamp window as typed, and none for other runs', () => { + const [integer, timestamp, plain] = toRuns( + [ + wire('i', { origin: 'backfill', window: { kind: 'integer', start: -3, end: 11 } }), + wire('t', { window: { kind: 'timestamp', start: '2026-10-01T00:00:00Z', end: '2026-10-31T00:00:00Z' } }), + wire('p') + ], + LOCATION + ); + expect(integer.window).toEqual({ start: '-3', end: '11' }); + expect(timestamp.window).toEqual({ start: '2026-10-01T00:00:00Z', end: '2026-10-31T00:00:00Z' }); + expect(plain.window).toBeNull(); + }); +}); + +describe('run queue', () => { + it('keeps claim order, drops runs that started meanwhile and never names a run elsewhere', () => { + const entries = queueEntriesOf( + [ + wire('urgent', { status: 'queued', queue: { position: 1, waits: [{ kind: 'concurrency_limit', limit: 'warehouse', holders: ['held'], elsewhere: 2 }] } }), + wire('started', { status: 'queued' }), + wire('blocked', { + status: 'queued', + queue: { + position: 2, + waits: [ + { kind: 'write_claim', run_id: null, node_key: 'model.orders', other_nodes: 3 }, + { kind: 'earlier_run', run_id: 'urgent', limit: 'warehouse' }, + { kind: 'upstream_job', run_id: 'stage', job_key: 'staging' }, + { kind: 'job_running', run_id: 'same' }, + { kind: 'write_claim', run_id: 'same', node_key: 'model.customers', other_nodes: 0 } + ] + } + }) + ], + LOCATION + ); + expect(entries.map((q) => [q.run.id, q.position])).toEqual([ + ['urgent', 1], + ['blocked', 2] + ]); + expect(entries[0].waits).toEqual([{ kind: 'concurrency_limit', limit: 'warehouse', holders: ['held'], elsewhere: 2 }]); + expect(entries[1].waits).toEqual([ + { kind: 'write_claim', runId: null, nodeKey: 'model.orders', otherNodes: 3 }, + { kind: 'earlier_run', runId: 'urgent', limit: 'warehouse' }, + { kind: 'upstream_job', runId: 'stage', jobKey: 'staging' }, + { kind: 'job_running', runId: 'same' } + ]); + }); +}); + +describe('runs past the newest window', () => { + const runWindow = (prefix: string, size: number) => Array.from({ length: size }, (_, i) => wire(`${prefix}${i}`)); + const ids = (page: { rows: { run: { id: string } }[] }) => page.rows.map((r) => r.run.id); + + it('reads the window count after the offset', () => { + expect(parseRunCursor(null)).toEqual({ offset: 0, older: 0 }); + expect(parseRunCursor('7')).toEqual({ offset: 7, older: 0 }); + expect(parseRunCursor('1000+2')).toEqual({ offset: 1000, older: 2 }); + }); + + it('stops at the end of a window shorter than a full one', () => { + const page = runRowPageOver([runWindow('n', 30)], LOCATION, { tab: 'all', limit: 20, cursor: '20' }); + expect(ids(page)).toEqual(Array.from({ length: 10 }, (_, i) => `n${i + 20}`)); + expect(page.nextCursor).toBeNull(); + }); + + it('asks for one more window once a full window runs out', () => { + const newest = runWindow('n', 1000); + const first = runRowPageOver([newest], LOCATION, { tab: 'all', limit: 600 }); + expect(first.nextCursor).toBe('600'); + const second = runRowPageOver([newest], LOCATION, { tab: 'all', limit: 600, cursor: first.nextCursor }); + expect(second.rows).toHaveLength(400); + expect(second.nextCursor).toBe('1000+1'); + }); + + it('pages into the older window and keeps its count on the cursor', () => { + const windows = [runWindow('n', 1000), runWindow('o', 1000)]; + const page = runRowPageOver(windows, LOCATION, { tab: 'all', limit: 600, cursor: '1000+1' }); + expect(ids(page)[0]).toBe('o0'); + expect(ids(page)).toHaveLength(600); + expect(page.nextCursor).toBe('1600+1'); + const last = runRowPageOver([runWindow('n', 1000), runWindow('o', 3)], LOCATION, { tab: 'all', limit: 600, cursor: '1000+1' }); + expect(ids(last)).toEqual(['o0', 'o1', 'o2']); + expect(last.nextCursor).toBeNull(); + }); +}); + describe('run detail', () => { it('links parent, children and reruns, and reruns from failed and skipped nodes but not cancelled ones', () => { const wires = [ @@ -189,3 +283,16 @@ describe('group run nodes', () => { ]); }); }); + +describe('planResultOf', () => { + it.each([ + { description: 'a plan keeps its models and fills missing sections', outcome: { kind: 'plan', document: { selected_count: 1, models: [{ name: 'orders', reason: 'first_run' }] } }, expected: { plan: 'orders', error: null } }, + { description: "sqb plan's own error is the result", outcome: { kind: 'plan', document: { error: { code: 'S007', message: "unknown selector name 'ordrs'" } } }, expected: { plan: null, error: "S007: unknown selector name 'ordrs'" } }, + { description: 'a superseded plan asks to plan again', outcome: { kind: 'superseded' }, expected: { plan: null, error: 'The project changed on the location while it was planning; plan again' } } + ])('$description', ({ outcome, expected }) => { + const result = planResultOf(outcome as never); + expect(result.error).toBe(expected.error); + expect(result.plan?.models[0]?.name ?? null).toBe(expected.plan); + if (result.plan) expect(result.plan.warnings).toEqual([]); + }); +}); diff --git a/ui/src/lib/api/http-runs.ts b/ui/src/lib/api/http-runs.ts index a5e2c0375b..c38991ec16 100644 --- a/ui/src/lib/api/http-runs.ts +++ b/ui/src/lib/api/http-runs.ts @@ -1,7 +1,7 @@ import type { components } from '$control/schema.d.ts'; import { isActiveRun } from '$lib/status'; import type { Capability, Principal } from './types'; -import type { AdhocRunCheck, LaunchVerb, NodeCounts, Run, RunDetail, RunListQuery, RunNode, RunNodeStatus, RunRow, RunRowPage, RunRowQuery, RunTabCounts, RunEvent, RunEventLevel, RunOutputLine, RunStatement } from './types/operate'; +import type { AdhocRunCheck, LaunchVerb, NodeCounts, PlanDocument, PlanResult, QueueEntry, QueueWait, Run, RunDetail, RunListQuery, RunNode, RunNodeStatus, RunRow, RunRowPage, RunRowQuery, RunTabCounts, RunEvent, RunEventLevel, RunOutputLine, RunStatement } from './types/operate'; import { RECALL_EVENT_TYPES, SCHEDULER_FACTS } from './types/operate'; type Wire = components['schemas']; @@ -11,6 +11,31 @@ export const RUN_WINDOW = 1000; const PERSON = 'principal:'; +/** The location's answer to a plan request: the plan, `sqb plan`'s own error, or a superseded plan. */ +export function planResultOf(outcome: Wire['LocationWorkOutcome']): PlanResult { + if (outcome.kind === 'superseded') return { plan: null, error: 'The project changed on the location while it was planning; plan again' }; + if (outcome.kind !== 'plan') return { plan: null, error: 'The location answered with something other than a plan' }; + const document = outcome.document as Partial & { error?: { code?: string; message?: string } }; + if (document.error) return { plan: null, error: [document.error.code, document.error.message].filter(Boolean).join(': ') }; + const list = (value: T[] | undefined): T[] => value ?? []; + return { + plan: { + selected_count: document.selected_count ?? 0, + models: list(document.models), + seeds: list(document.seeds), + source_loads: list(document.source_loads), + functions: list(document.functions), + python_nodes: list(document.python_nodes), + warnings: list(document.warnings), + migrations: list(document.migrations), + column_migrations: list(document.column_migrations), + table_type_conversions: list(document.table_type_conversions), + retention: list(document.retention) + }, + error: null + }; +} + /** The server's dry run of an ad-hoc command. */ export function toAdhocCheck(wire: Wire['AdhocCheck']): AdhocRunCheck { return { @@ -66,6 +91,8 @@ export function toRun(wire: Wire['RunRecord'], locationId: string, childRunCount tickScheduledFor: null, runKey: null, rerunOfRunId: wire.rerun_of_run_id ?? null, + narrowing: wire.narrowing ?? null, + window: wire.window ? { start: String(wire.window.start), end: String(wire.window.end) } : null, requestedBy: requesterOf(wire), priority: wire.priority, rewriteHistory: wire.rewrite_history, @@ -83,6 +110,34 @@ export function toRun(wire: Wire['RunRecord'], locationId: string, childRunCount } /** Runs keyed by id, each with its number of child runs in the listing. */ +/** The queue in claim order; a run the read sent without queue state (it started meanwhile) is left out. */ +export function queueEntriesOf(wires: Wire['RunRecord'][], locationId: string): QueueEntry[] { + const runs = toRuns(wires, locationId); + return wires.flatMap((wire, i) => (wire.queue ? [{ run: runs[i], position: wire.queue.position, waits: shownWaits(wire.queue.waits.map(toQueueWait)), overlap: null }] : [])); +} + +/** A running run of the same job also holds the job's write claims; name it once. */ +function shownWaits(waits: QueueWait[]): QueueWait[] { + const sameJob = new Set(waits.flatMap((w) => (w.kind === 'job_running' ? [w.runId] : []))); + return waits.filter((w) => w.kind !== 'write_claim' || !w.runId || !sameJob.has(w.runId)); +} + +function toQueueWait(wait: Wire['QueueWait']): QueueWait { + switch (wait.kind) { + case 'write_claim': + return { kind: wait.kind, runId: wait.run_id ?? null, nodeKey: wait.node_key, otherNodes: wait.other_nodes }; + case 'concurrency_limit': + return { kind: wait.kind, limit: wait.limit, holders: wait.holders, elsewhere: wait.elsewhere }; + case 'earlier_run': + return { kind: wait.kind, runId: wait.run_id ?? null, limit: wait.limit }; + case 'upstream_job': + return { kind: wait.kind, runId: wait.run_id, jobKey: wait.job_key }; + case 'job_running': + case 'group_parent': + return { kind: wait.kind, runId: wait.run_id }; + } +} + export function toRuns(wires: Wire['RunRecord'][], locationId: string): Run[] { const children = new Map(); for (const w of wires) if (w.parent_run_id) children.set(w.parent_run_id, (children.get(w.parent_run_id) ?? 0) + 1); @@ -170,6 +225,22 @@ export function runRowPage(wires: Wire['RunRecord'][], locationId: string, query }; } +/** A rows cursor: the offset into the matching rows, then `+N` when N windows older than the newest are loaded. */ +export function parseRunCursor(cursor: string | null | undefined): { offset: number; older: number } { + const [offset, older] = (cursor ?? '0').split('+'); + return { offset: Number(offset) || 0, older: Number(older) || 0 }; +} + +/** A page over the loaded windows, newest first; once they run out and the oldest was full, the next page loads one more window. */ +export function runRowPageOver(windows: Wire['RunRecord'][][], locationId: string, query: RunRowQuery): RunRowPage { + const { offset, older } = parseRunCursor(query.cursor); + const page = runRowPage(windows.flat(), locationId, { ...query, cursor: String(offset) }); + const more = windows.at(-1)?.length === RUN_WINDOW; + const next = page.nextCursor ?? (more ? String(offset + page.rows.length) : null); + const loaded = page.nextCursor ? older : older + 1; + return { rows: page.rows, nextCursor: next !== null && loaded ? `${next}+${loaded}` : next }; +} + export function runTabCountsOf(wires: Wire['RunRecord'][], locationId: string, query: RunRowQuery): RunTabCounts { const runs = toRuns(wires, locationId).filter((run, i) => matches(run, wires[i], query)); return { diff --git a/ui/src/lib/api/http-timeline.test.ts b/ui/src/lib/api/http-timeline.test.ts new file mode 100644 index 0000000000..ce7e35314d --- /dev/null +++ b/ui/src/lib/api/http-timeline.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest'; +import type { components } from '$control/schema.d.ts'; +import { timelineOf } from './http-timeline'; + +type W = components['schemas']; + +const LOCATION = '0191f0aa-orders:development'; +const WINDOW = { since: '2026-11-09T08:00:00Z', until: '2026-11-09T12:00:00Z' }; +const NOW = Date.parse('2026-11-09T11:00:00Z'); + +function run(id: string, overrides: Partial): W['RunRecord'] { + return { + run_id: id, + root_run_id: id, + kind: 'job', + origin: 'schedule', + status: 'success', + job_key: 'orders_daily', + last_event_storage_id: 1, + location_id: '0191f0aa-0000-7000-8000-000000000000', + node_counts_json: {}, + priority: 0, + queued_at: '2026-11-09T09:00:00Z', + requested_by: 'system:scheduler', + rewrite_history: false, + selection_json: ['model.orders'], + snapshot_id: '0191f0dd-0000-7000-8000-000000000000', + target: 'development', + ...overrides + }; +} + +function schedule(id: string, job: string, overrides: Partial = {}): W['TriggerRecord'] { + return { + trigger_id: id, + name: `${job}_hourly`, + job_key: job, + status: 'running', + definition_json: { description: '', job_key: job, schedule: { cron: '0 * * * *', timezone: 'UTC' } } as unknown as W['TriggerDeclaration'], + ...overrides + } as W['TriggerRecord']; +} + +const jobs = [ + { job_key: 'orders_daily', kind: 'job' }, + { job_key: 'nightly', kind: 'group' } +] as W['JobRecord'][]; + +describe('timeline', () => { + const timeline = timelineOf( + { + runs: [ + run('ten-min', { started_at: '2026-11-09T09:00:00Z', ended_at: '2026-11-09T09:10:00Z' }), + run('two-min', { started_at: '2026-11-09T10:00:00Z', ended_at: '2026-11-09T10:02:00Z' }), + run('four-min', { started_at: '2026-11-09T10:30:00Z', ended_at: '2026-11-09T10:34:00Z' }), + run('failed-long', { status: 'failed', started_at: '2026-11-09T09:20:00Z', ended_at: '2026-11-09T09:50:00Z' }), + run('running', { status: 'running', started_at: '2026-11-09T10:50:00Z' }), + run('before', { started_at: '2026-11-09T07:00:00Z', ended_at: '2026-11-09T07:30:00Z' }), + run('child', { parent_run_id: 'p', root_run_id: 'p', started_at: '2026-11-09T09:00:00Z', ended_at: '2026-11-09T09:01:00Z' }), + run('queued', { status: 'queued' }) + ], + jobs, + triggers: [ + schedule('t1', 'orders_daily', { next_due_at: '2026-11-09T11:30:00Z' }), + schedule('t2', 'nightly', { next_due_at: '2026-11-09T11:45:00Z' }), + schedule('t3', 'orders_daily', { status: 'paused', next_due_at: '2026-11-09T11:15:00Z' }), + schedule('t4', 'orders_daily', { next_due_at: '2026-11-09T13:00:00Z' }) + ], + ticks: [ + { tick_id: 'k1', trigger_id: 't1', outcome: 'covered', covered_by_run_id: 'running', scheduled_for: '2026-11-09T10:55:00Z', epoch: 1, evaluated_at: '2026-11-09T10:55:01Z' }, + { tick_id: 'k2', trigger_id: 't1', outcome: 'requested', run_id: 'two-min', scheduled_for: '2026-11-09T10:00:00Z', epoch: 1, evaluated_at: '2026-11-09T10:00:01Z' }, + { tick_id: 'k3', trigger_id: 't1', outcome: 'covered', scheduled_for: '2026-11-09T07:55:00Z', epoch: 1, evaluated_at: '2026-11-09T07:55:01Z' } + ] as W['TickRecord'][] + }, + LOCATION, + WINDOW, + NOW + ); + + it('shows started top-level runs overlapping the window, a running one up to now', () => { + const real = timeline.bars.filter((b) => b.runId); + expect(real.map((b) => b.runId)).toEqual(['ten-min', 'failed-long', 'two-min', 'four-min', 'running']); + expect(real.at(-1)?.end).toBe('2026-11-09T11:00:00.000Z'); + }); + + it("projects each running schedule's next tick inside the window, as long as the job's median successful run", () => { + const projected = timeline.bars.filter((b) => b.status === 'projected'); + expect(projected).toEqual([ + { runId: null, jobKey: 'orders_daily', kind: 'job', origin: 'schedule', status: 'projected', triggerName: 'orders_daily_hourly', start: '2026-11-09T11:30:00Z', end: '2026-11-09T11:34:00.000Z' }, + { runId: null, jobKey: 'nightly', kind: 'group', origin: 'schedule', status: 'projected', triggerName: 'nightly_hourly', start: '2026-11-09T11:45:00Z', end: '2026-11-09T11:46:00.000Z' } + ]); + }); + + it('marks covered ticks inside the window only', () => { + expect(timeline.markers.map((m) => [m.tickId, m.coveredByRunId, m.jobKey])).toEqual([['k1', 'running', 'orders_daily']]); + expect(timeline.jobKeys).toEqual(['orders_daily', 'nightly']); + }); +}); diff --git a/ui/src/lib/api/http-timeline.ts b/ui/src/lib/api/http-timeline.ts new file mode 100644 index 0000000000..aefade2a02 --- /dev/null +++ b/ui/src/lib/api/http-timeline.ts @@ -0,0 +1,68 @@ +import type { components } from '$control/schema.d.ts'; +import { toRuns } from './http-runs'; +import type { Timeline, TimelineBar, TimelineTickMarker } from './types/operate'; + +type Wire = components['schemas']; + +/** A projected tick's length when its job has no finished successful run to go by. */ +const FALLBACK_TYPICAL_MS = 60_000; + +export type TimelineReads = { + runs: Wire['RunRecord'][]; + jobs: Wire['JobRecord'][]; + triggers: Wire['TriggerRecord'][]; + ticks: Wire['TickRecord'][]; +}; + +/** Top-level job runs overlapping the window, each running schedule's next due tick, and ticks an existing run covered. */ +export function timelineOf(reads: TimelineReads, locationId: string, window: { since: string; until: string }, now: number): Timeline { + const nowIso = new Date(now).toISOString(); + const bars: TimelineBar[] = toRuns(reads.runs, locationId).flatMap((r) => { + if (r.parentRunId || !r.jobKey || !r.startedAt) return []; + const end = r.endedAt ?? nowIso; + if (end <= window.since || r.startedAt >= window.until) return []; + return [{ runId: r.id, jobKey: r.jobKey, kind: r.kind, origin: r.origin, status: r.status, triggerName: r.triggerName, start: r.startedAt, end }]; + }); + const kinds = new Map(reads.jobs.map((j) => [j.job_key, j.kind])); + for (const t of reads.triggers) { + const due = t.next_due_at; + if (!('schedule' in t.definition_json) || t.status !== 'running' || !due || due < window.since || due >= window.until) continue; + bars.push({ + runId: null, + jobKey: t.job_key, + kind: kinds.get(t.job_key) === 'group' ? 'group' : 'job', + origin: 'schedule', + status: 'projected', + triggerName: t.name, + start: due, + end: new Date(Date.parse(due) + typicalMs(bars, t.job_key)).toISOString() + }); + } + const triggers = new Map(reads.triggers.map((t) => [t.trigger_id, t])); + const markers: TimelineTickMarker[] = reads.ticks.flatMap((tick) => { + const trigger = triggers.get(tick.trigger_id); + if (!trigger || tick.outcome !== 'covered' || tick.scheduled_for < window.since || tick.scheduled_for >= window.until) return []; + return [ + { + tickId: tick.tick_id, + triggerId: tick.trigger_id, + triggerName: trigger.name, + jobKey: trigger.job_key, + scheduledFor: tick.scheduled_for, + outcome: 'covered', + coveredByRunId: tick.covered_by_run_id ?? null, + reason: null + } + ]; + }); + return { now: nowIso, jobKeys: reads.jobs.map((j) => j.job_key), bars: bars.sort((a, b) => a.start.localeCompare(b.start)), markers }; +} + +/** The median length of the job's successful runs on the timeline. */ +function typicalMs(bars: TimelineBar[], jobKey: string): number { + const lengths = bars + .filter((b) => b.jobKey === jobKey && b.status === 'success') + .map((b) => Date.parse(b.end) - Date.parse(b.start)) + .sort((a, b) => a - b); + return lengths.length ? lengths[Math.floor(lengths.length / 2)] : FALLBACK_TYPICAL_MS; +} diff --git a/ui/src/lib/api/mock/mock-client.ts b/ui/src/lib/api/mock/mock-client.ts index f84884dfcc..6aef706b00 100644 --- a/ui/src/lib/api/mock/mock-client.ts +++ b/ui/src/lib/api/mock/mock-client.ts @@ -3,6 +3,7 @@ import { ApiError } from '../client'; import { LocalChangeBus, type ConnectionState } from '../signals'; import type { AdhocRunCheck, + PlanResult, AdhocRunRequest, Bootstrap, BuiltInRole, @@ -266,6 +267,11 @@ export class MockApiClient implements ApiClient { return this.#operate.launchAdhocRun(scope, request); } + async planBuild(scope: Scope, command: string[]): Promise { + this.#location(scope); + return this.#operate.planBuild(scope, command); + } + async listJobs(scope: Scope): Promise { this.#location(scope); return this.#operate.listJobs(scope); @@ -352,8 +358,8 @@ export class MockApiClient implements ApiClient { revokeLocationToken(...args: Parameters) { return this.#platform.revokeLocationToken(...args); } - listWarehouseConnections(...args: Parameters) { - return this.#platform.listWarehouseConnections(...args); + listWarehouseConnections() { + return this.#platform.listWarehouseConnections(); } listProviders(...args: Parameters) { return this.#platform.listProviders(...args); diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index 4723e9ddc0..8afd687b26 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -9,6 +9,7 @@ import type { LocalChangeBus } from '../signals'; import type { AdhocPermission, AdhocRunCheck, + PlanResult, AdhocRunRequest, BuiltInRole, BulkRunResult, @@ -482,6 +483,7 @@ export class MockOperate { const user = this.#ctx.user(); const id = this.#newId(); if (job.kind === 'group') { + if (options.select?.length || options.exclude?.length) throw new ApiError('invalid control-plane data: a job group cannot be narrowed; narrow one of its jobs', 400); const parent = this.#create({ id, origin: 'manual', jobKey: key, by: user, select: [], exclude: [], flags, nodes: [], priority: job.priority }); job.members.forEach((m, i) => { const member = jobDef(m)!; @@ -491,7 +493,13 @@ export class MockOperate { this.#ctx.changes.emit('runs'); return this.#ctx.reply(this.#view(parent, scope)); } - const rec = this.#create({ id, origin: 'manual', jobKey: key, by: user, select: selectorTerms(job), exclude: job.exclude, flags, nodes: jobNodes(key), priority: job.priority }); + const named = (names: string[] | undefined, node: string) => !!names?.includes(node.slice(node.indexOf(':') + 1)); + const narrowed = !!(options.select?.length || options.exclude?.length); + if (narrowed) this.#need('view'); + const nodes = jobNodes(key).filter((n) => (!options.select?.length || named(options.select, n)) && !named(options.exclude, n)); + if (narrowed && !nodes.length) throw new ApiError("invalid control-plane data: the selectors leave none of the job's nodes", 400); + const narrowing = narrowed ? { select: options.select ?? [], exclude: options.exclude ?? [] } : null; + const rec = this.#create({ id, origin: 'manual', jobKey: key, narrowing, by: user, select: selectorTerms(job), exclude: job.exclude, flags, nodes, priority: job.priority }); this.#ctx.changes.emit('runs'); return this.#ctx.reply(this.#view(rec, scope)); } @@ -522,6 +530,27 @@ export class MockOperate { return this.#ctx.reply(this.#view(rec, scope)); } + /** Every selected model is planned as unchanged unless the command asks for a full refresh. */ + async planBuild(scope: Scope, command: string[]): Promise { + const check = this.#check(scope, { command, priority: 0 }); + await new Promise((resolve) => setTimeout(resolve, 1200)); + if (check.error) return this.#ctx.reply({ plan: null, error: check.error }); + if (check.verb !== 'build') return this.#ctx.reply({ plan: null, error: `sqb plan previews builds; sqb ${check.verb} cannot be planned` }); + const reason = command.includes('--full-refresh') ? 'full_refresh' : 'no_change'; + const models = check.selection.filter((key) => key.startsWith('model:')).map((key) => ({ + name: key.slice('model:'.length), + materialization_type: 'table', + action: 'create_table', + reason, + backfill: { action: reason === 'full_refresh' ? 'full' : 'forward', duration: null }, + qualified_name: `main.${key.slice('model:'.length)}` + })); + return this.#ctx.reply({ + plan: { selected_count: check.selection.length, models, seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [] }, + error: null + }); + } + // ---- jobs ---- listJobs(scope: Scope): Promise { @@ -749,7 +778,7 @@ export class MockOperate { } const waits = this.#queue(); for (const rec of this.#ordered()) { - if ((waits.get(rec.run.id) ?? []).some((w) => w.kind !== 'priority')) continue; + if ((waits.get(rec.run.id) ?? []).some((w) => w.kind !== 'earlier_run')) continue; rec.run.status = 'launched'; rec.run.launchedAt = new Date(now).toISOString(); if (!rec.script.nodes.length) { @@ -796,20 +825,22 @@ export class MockOperate { for (const h of holders) { const overlap = h.script.nodes.filter((n) => mine.has(n)); if (overlap.length) - waits.push({ kind: 'write_claim', runId: h.run.id, jobKey: h.run.jobKey, nodeKey: overlap[0], otherNodes: overlap.length - 1 }); + waits.push({ kind: 'write_claim', runId: h.run.id, nodeKey: overlap[0], otherNodes: overlap.length - 1 }); } const fullGroups: string[] = []; for (const group of rec.script.concurrencyGroups) { const inGroup = holders.filter((h) => h.script.concurrencyGroups.includes(group)).map((h) => h.run.id); const limit = CONCURRENCY_LIMITS[group] ?? 1; if (inGroup.length >= limit) { - waits.push({ kind: 'concurrency_group', group, limit, holders: inGroup }); + waits.push({ kind: 'concurrency_limit', limit: group, holders: inGroup, elsewhere: 0 }); fullGroups.push(group); } } if (fullGroups.length) { - const before = ahead.filter((a) => a.script.concurrencyGroups.some((g) => fullGroups.includes(g))).map((a) => a.run.id); - if (before.length) waits.push({ kind: 'priority', aheadRunIds: before }); + for (const group of fullGroups) { + const first = ahead.find((a) => a.script.concurrencyGroups.includes(group)); + if (first) waits.push({ kind: 'earlier_run', runId: first.run.id, limit: group }); + } } if (rec.run.parentRunId) { const parent = this.#byId(rec.run.parentRunId); @@ -822,7 +853,7 @@ export class MockOperate { if (sibling && isActive(sibling.run.status)) waits.push({ kind: 'upstream_job', runId: sibling.run.id, jobKey: up.jobKey }); } } - if (parent && parent.run.status === 'queued') waits.push({ kind: 'group_parent', runId: parent.run.id, jobKey: parent.run.jobKey ?? '' }); + if (parent && parent.run.status === 'queued') waits.push({ kind: 'group_parent', runId: parent.run.id }); } out.set(rec.run.id, waits); ahead.push(rec); @@ -858,6 +889,7 @@ export class MockOperate { id: string; origin: Run['origin']; jobKey: string | null; + narrowing?: Run['narrowing']; by: Principal; select: string[]; exclude: string[]; @@ -887,6 +919,8 @@ export class MockOperate { tickScheduledFor: null, runKey: null, rerunOfRunId: spec.rerunOf ?? null, + narrowing: spec.narrowing ?? null, + window: null, requestedBy: spec.by, priority: spec.priority, rewriteHistory: Object.keys(spec.flags).some((f) => flagSpec(spec.verb ?? 'build', f)?.requires === 'run.rewrite_history'), diff --git a/ui/src/lib/api/mock/operate.ts b/ui/src/lib/api/mock/operate.ts index c500962542..15ebe70c1c 100644 --- a/ui/src/lib/api/mock/operate.ts +++ b/ui/src/lib/api/mock/operate.ts @@ -776,6 +776,8 @@ export function createOperateFixtures(now: number): OperateFixtures { tickScheduledFor: tick != null ? new Date(tick).toISOString() : null, runKey: seed.runKey ?? null, rerunOfRunId: seed.rerunOf ?? null, + narrowing: null, + window: null, requestedBy: seed.by, priority: seed.priority ?? job?.priority ?? 0, rewriteHistory: '--full-refresh' in flags || Object.keys(flags).some((f) => f.includes('cursor')) || '--reload' in flags, @@ -1000,7 +1002,6 @@ export function createOperateFixtures(now: number): OperateFixtures { const fingerprint = (name: string, version = 1) => `fp:${name.length.toString(16)}${version}a${(name.charCodeAt(0) * 7919).toString(16)}`; const evalOf = (name: string, seq: number, at: number, outcome: SensorEvaluation['outcome'], extra: Partial = {}): SensorEvaluation => ({ id: `eval-${name.replace(/_/g, '-')}-${seq}`, - sequence: seq, outcome, requestedAt: new Date(at).toISOString(), finishedAt: outcome === 'requested' ? null : new Date(at + 1400).toISOString(), diff --git a/ui/src/lib/api/mock/platform.ts b/ui/src/lib/api/mock/platform.ts index ebafb48a60..944f0e0cbb 100644 --- a/ui/src/lib/api/mock/platform.ts +++ b/ui/src/lib/api/mock/platform.ts @@ -46,6 +46,18 @@ import type { SsoMapping, WarehouseConnection } from '../types'; + +/** A warehouse connection as the fixtures declare it; the client sees only what its checks and syncs report. */ +type MockWarehouse = { + name: string; + target: string; + adapter: string; + purpose: 'target' | 'catalog'; + sourceFile: string; + settings: SettingField[]; + values: Record; + lastTest: ConnectionTest | null; +}; import { ADMIN_ONLY, baseGrantsOf, @@ -305,7 +317,7 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { ], values: { account: 'example-account', database, warehouse: 'TRANSFORMING_WH' } }); - const warehouseByLocation: Record = { + const warehouseByLocation: Record = { [LOCATION_SHOP]: [ { name: 'prod', target: 'prod', adapter: 'snowflake', purpose: 'target', sourceFile: 'sqlbuild.toml', ...snowflakeSettings('WAFFLE_SHOP'), lastTest: locationExtras[LOCATION_SHOP].connectionTest }, { @@ -679,12 +691,21 @@ export class MockPlatformApi implements PlatformApi { // ---- connections ---- - async listWarehouseConnections(scope: Scope): Promise { - const warehouse = this.#warehouse(scope).map((c) => ({ - ...c, - lastTest: this.#connectionTests.get(`${scope.locationId}/${c.name}`) ?? c.lastTest - })); - return this.#ctx.reply(structuredClone(warehouse)); + async listWarehouseConnections(): Promise { + const synced = new Date(Date.now() - 4 * 60_000).toISOString(); + const connections = this.#ctx.locations.flatMap((l) => + this.#warehouse({ locationId: l.id }) + .filter((c) => c.purpose === 'target') + .map((c) => ({ + id: l.id, + location: l.name, + target: c.target, + stateLocation: { database: typeof c.values.database === 'string' ? c.values.database : null, schema: 'sqlbuild' }, + lastTest: this.#connectionTests.get(`${l.id}/${c.name}`) ?? c.lastTest, + tables: ['audit_results', 'catalog', 'fingerprints', 'statistics'].map((table, i) => ({ table, at: synced, recordsSent: 12 * (i + 1), elapsedMs: 180 + 40 * i, failure: null, lastSyncedAt: synced })) + })) + ); + return this.#ctx.reply(structuredClone(connections)); } async listProviders(scope: Scope): Promise { @@ -882,9 +903,9 @@ export class MockPlatformApi implements PlatformApi { return this.#ctx.reply({ serverVersion: this.#ctx.serverVersion, storage: 'postgresql', - schemaVersion: 14, - latestSchemaVersion: 14, - schemaMigratedAt: new Date(this.#ctx.now - 5 * DAY).toISOString(), + installedAt: new Date(this.#ctx.now - 120 * DAY).toISOString(), + schemaChecksum: '6f1c2a9e4b7d0c3f8a5e2b1d9c4f7a0e3b6d8c1f5a2e9b4d7c0f3a6e1b8d5c2f', + schemaAppliedAt: new Date(this.#ctx.now - 5 * DAY).toISOString(), authMode: 'oidc', schedulerEnabled: true, retention: [ @@ -913,7 +934,7 @@ export class MockPlatformApi implements PlatformApi { }); } - #warehouse(scope: Scope): WarehouseConnection[] { + #warehouse(scope: Scope): MockWarehouse[] { return this.#data.warehouseByLocation[scope.locationId] ?? []; } diff --git a/ui/src/lib/api/platform-client.ts b/ui/src/lib/api/platform-client.ts index ccbacb8dd7..5041fc5cbb 100644 --- a/ui/src/lib/api/platform-client.ts +++ b/ui/src/lib/api/platform-client.ts @@ -57,8 +57,8 @@ export interface PlatformApi { revokeLocationToken(tokenId: string): Promise; // ---- warehouse connections and providers ---- - /** Warehouse connections of the selected target. */ - listWarehouseConnections(scope: Scope): Promise; + /** The warehouse connection of every location target the person may list whole. */ + listWarehouseConnections(): Promise; /** Providers declared in the project, with where each one is used. Read-only. */ listProviders(scope: Scope): Promise; diff --git a/ui/src/lib/api/types.ts b/ui/src/lib/api/types.ts index da588447be..3349a87cc6 100644 --- a/ui/src/lib/api/types.ts +++ b/ui/src/lib/api/types.ts @@ -110,6 +110,7 @@ export type Bootstrap = { export type PlatformEventKind = | 'tick_overdue' + | 'calendar_horizon_low' | 'scheduler_no_leader' | 'location_broken' | 'launch_failed' diff --git a/ui/src/lib/api/types/build.ts b/ui/src/lib/api/types/build.ts index 0e2b541e0b..e86a845cf8 100644 --- a/ui/src/lib/api/types/build.ts +++ b/ui/src/lib/api/types/build.ts @@ -116,14 +116,17 @@ export type WarehouseColumnState = 'match' | 'type_differs' | 'missing' | 'undec /** The synced information-schema object and its sampled statistics. */ export type WarehouseObject = { relation: RelationName; - objectType: 'table' | 'view'; - createdAt: Iso; - lastAlteredAt: Iso; + /** As the warehouse names it, such as `table` or `view`. */ + objectType: string; + /** Null where the sync does not record creation or alteration times. */ + createdAt: Iso | null; + lastAlteredAt: Iso | null; rows: number | null; bytes: number | null; - columns: { name: string; declaredType: string | null; actualType: string | null; state: WarehouseColumnState }[]; + /** `state` is null when declared types are not known, so nothing is compared. */ + columns: { name: string; declaredType: string | null; actualType: string | null; state: WarehouseColumnState | null }[]; /** Sampled series, oldest first. */ - stats: { at: Iso; rows: number; bytes: number }[]; + stats: { at: Iso; rows: number; bytes: number | null }[]; syncedAt: Iso; }; @@ -136,7 +139,7 @@ export type MicrobatchState = 'complete' | 'gap' | 'replay_required'; export type MicrobatchCoverage = { cursorColumn: string; /** Bounds are timestamp text for `timestamp` cursors and integer text for `integer` cursors. */ - cursorType: 'timestamp' | 'integer'; + cursorType: string; batchSize: string; /** Start of the earliest completed partition. */ earliest: string; diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index c81c6679dd..62970a7541 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -67,6 +67,10 @@ export type Run = { /** Sensor run requests are de-duplicated by run key. */ runKey: string | null; rerunOfRunId: string | null; + /** How a person narrowed this run of a job, as typed; null when it runs the whole job. */ + narrowing: { select: string[]; exclude: string[] } | null; + /** The cursor window a backfill processes, inclusive; null for every other run. */ + window: { start: string; end: string } | null; requestedBy: Principal; priority: number; /** Full refresh or an explicit cursor window was requested. */ @@ -387,13 +391,17 @@ export type RunDetail = { // ---- queue ---- +/** Why the scheduler can't claim a queued run yet; a run in another location or target is never named (`runId` null). */ export type QueueWait = - | { kind: 'write_claim'; runId: string; jobKey: string | null; nodeKey: string; otherNodes: number } - | { kind: 'concurrency_group'; group: string; limit: number; holders: string[] } - | { kind: 'priority'; aheadRunIds: string[] } + | { kind: 'write_claim'; runId: string | null; nodeKey: string; otherNodes: number } + /** Running runs hold slots of a concurrency limit this run takes; `elsewhere` counts holders in other locations or targets. */ + | { kind: 'concurrency_limit'; limit: string; holders: string[]; elsewhere: number } + /** An earlier queued run takes the same limit, so it goes first. */ + | { kind: 'earlier_run'; runId: string | null; limit: string } + | { kind: 'job_running'; runId: string } | { kind: 'upstream_job'; runId: string; jobKey: string } - /** A child run waits for its group run to be claimed. */ - | { kind: 'group_parent'; runId: string; jobKey: string }; + /** A child run waits for its group run to start. */ + | { kind: 'group_parent'; runId: string }; export type OverlapOutcome = { policy: OverlapPolicy; @@ -485,6 +493,40 @@ export type AdhocRunCheck = { launchable: boolean; }; +/** One model in `sqb plan --json`: what the build would do to it and why. */ +export type PlanModel = { + name: string; + materialization_type: string; + action: string; + reason: string; + incremental_strategy?: string; + cursor_column?: string; + cursor_bounds?: { start: string | null; end: string | null } | null; + backfill: { action: string; duration: string | null }; + qualified_name?: string; + changed_functions?: string[]; +}; + +export type PlanWarning = { severity: string; message: string; model_name?: string; code?: string }; + +/** `sqb plan --json` as the location printed it; sections the page doesn't show are kept as they are. */ +export type PlanDocument = { + selected_count: number; + models: PlanModel[]; + seeds: { name: string; reason: string; qualified_name?: string }[]; + source_loads: { name: string; kind: string; write_strategy?: string; cursor_column?: string; is_reload: boolean }[]; + functions: { name: string; language: string; reason?: string }[]; + python_nodes: { name: string; kind: string; phase: string; identity_status: string }[]; + warnings: PlanWarning[]; + migrations: { model: string; origin_model?: string; decision: string; completed_at: string | null }[]; + column_migrations: { model: string; origin_column: string; destination_column: string; decision: string }[]; + table_type_conversions: { model: string; actual_type?: string; desired_type?: string }[]; + retention: { model: string; action?: string }[]; +}; + +/** The location's plan of a build command, or why it could not plan it. */ +export type PlanResult = { plan: PlanDocument; error: null } | { plan: null; error: string }; + /** Cursor window for a launch, typed from the selected nodes' cursor type. */ export type CursorWindow = { type: 'timestamp' | 'integer'; @@ -494,6 +536,10 @@ export type CursorWindow = { export type JobLaunchOptions = { window?: CursorWindow; + /** Runs only the job's nodes these `sqb build` selectors also select; the run stays a run of the job. */ + select?: string[]; + /** Leaves the job's nodes these selectors select out of the run. */ + exclude?: string[]; }; /** `selected` reruns exactly the nodes passed with the request. */ @@ -635,7 +681,6 @@ export type SensorEvaluationOutcome = 'requested' | 'completed' | 'failed' | 'ti export type SensorEvaluation = { id: string; - sequence: number; /** The latest phase recorded; `requested` alone means in flight. */ outcome: SensorEvaluationOutcome; requestedAt: Iso; diff --git a/ui/src/lib/api/types/platform.ts b/ui/src/lib/api/types/platform.ts index 823fbe6630..b374e8b8b0 100644 --- a/ui/src/lib/api/types/platform.ts +++ b/ui/src/lib/api/types/platform.ts @@ -215,17 +215,28 @@ export type SettingField = { secretSource?: string; }; +/** A location target's warehouse connection as its checks and syncs report it; locations publish no connection settings. */ export type WarehouseConnection = { - name: string; + /** The location key, `location_id:target`. */ + id: string; + location: string; target: string; - adapter: string; - /** `catalog_connection` override used by the warehouse sync, if any. */ - purpose: 'target' | 'catalog'; - sourceFile: string; - settings: SettingField[]; - /** Configured values; secrets are never returned. */ - values: Record; + /** Where SQLBuild's own tables are read back from; null before a sync reports it. */ + stateLocation: { database: string | null; schema: string } | null; lastTest: ConnectionTest | null; + /** The newest sync of each SQLBuild table, in table order. */ + tables: WarehouseTableSync[]; +}; + +export type WarehouseTableSync = { + table: string; + at: Iso; + recordsSent: number; + elapsedMs: number; + /** What the newest sync was doing when it failed; null when it succeeded. */ + failure: string | null; + /** The newest sync of the table that succeeded. */ + lastSyncedAt: Iso | null; }; /** A connection to something other than the warehouse (an API, object storage, a chat webhook), declared in project code. */ @@ -515,10 +526,12 @@ export type RetentionSetting = { name: string; value: string; description: strin export type ServerSettings = { serverVersion: string; storage: 'postgresql' | 'sqlite'; - schemaVersion: number; - latestSchemaVersion: number; - schemaMigratedAt: Iso; + installedAt: Iso; + /** A server starts only on the exact schema it creates, so a running server's schema is always current. */ + schemaChecksum: string; + schemaAppliedAt: Iso; authMode: AuthMode; + /** Whether this server runs the scheduler; another server may hold the lease. */ schedulerEnabled: boolean; retention: RetentionSetting[]; }; diff --git a/ui/src/lib/components/admin/access-checker.svelte b/ui/src/lib/components/admin/access-checker.svelte index 788c70ed01..1233d68b55 100644 --- a/ui/src/lib/components/admin/access-checker.svelte +++ b/ui/src/lib/components/admin/access-checker.svelte @@ -1,4 +1,5 @@ diff --git a/ui/src/lib/components/platform/problem-list.svelte b/ui/src/lib/components/platform/problem-list.svelte index ea96a81596..27c5f6ae18 100644 --- a/ui/src/lib/components/platform/problem-list.svelte +++ b/ui/src/lib/components/platform/problem-list.svelte @@ -1,4 +1,5 @@ @@ -67,7 +70,7 @@ {#if otherPages.length}

Pages you can use:

{/if} {#if (app.bootstrap?.locations.length ?? 0) > 1}

Or choose another location from the menu at the top of the sidebar.

{/if} @@ -77,11 +80,16 @@
- {#if issues.length}{:else}{/if} + {#if loaded && issues.length}{:else}{/if}
- {issues.length ? `${app.location?.name} needs attention` : `${app.location?.name} is healthy`} - {issues.length ? `${issues.length} ${issues.length === 1 ? 'item' : 'items'} · ${latestText}` : latestText} + {#if !loaded} + {app.location?.name} + {overview.error ? 'Couldn’t check its health' : 'Checking…'} + {:else} + {issues.length ? `${app.location?.name} needs attention` : `${app.location?.name} is healthy`} + {issues.length ? `${issues.length} ${issues.length === 1 ? 'item' : 'items'} · ${latestText}` : latestText} + {/if}
{#if m}
@@ -90,7 +98,7 @@ {#if m.checks.failing}{m.checks.failing} failing {m.checks.failing === 1 ? 'check' : 'checks'}{/if}
{/if} - {#if latest}View latest run →{/if} + {#if latest}View latest run →{/if}
{#if issues.length}
@@ -98,14 +106,14 @@ {@const Glyph = attentionIcon[view.tone]}
- {view.title} + {view.title} {view.detail} {view.since} {#if item.kind === 'stale_models'} - goto(planHref(staleSelection))}>Rebuild {staleSelection.length} + goto(to(planHref(staleSelection)))}>Rebuild {staleSelection.length} {/if} - {view.action} → + {view.action} →
{/each} @@ -151,7 +159,7 @@

Activity

Past runs, and when schedules will run next
- Open full timeline → + Open full timeline →
@@ -161,12 +169,12 @@ {#if showGroups}

Build status by group

Current, stale and failed nodes · {app.location?.name}
- Open catalog → + Open catalog →
GroupCurrentChangedBehindFailed
{#each groups as g (g.group)} - + {g.group} @@ -186,11 +194,11 @@

Recent changes

Recent deploys
- All changes → + All changes →
{#each data?.recentChanges ?? [] as c (c.snapshotId)} - + {c.gitRevision?.slice(0, 7) ?? '—'}version {c.generation} {summarizeChange(c)} diff --git a/ui/src/routes/admin/audit/+page.svelte b/ui/src/routes/admin/audit/+page.svelte index b1412ebbb0..93f431ebce 100644 --- a/ui/src/routes/admin/audit/+page.svelte +++ b/ui/src/routes/admin/audit/+page.svelte @@ -1,4 +1,5 @@ @@ -89,7 +90,7 @@
- +

{shortName(n?.key ?? name)}

@@ -104,7 +105,7 @@ {/if}
- View in lineage + View in lineage
{#if n} @@ -143,13 +144,13 @@ {:else if tab === 'sql'} {#if n.sql} - goto(`/catalog/${m}?tab=sql`)} /> + goto(to(`/catalog/${m}?tab=sql`))} /> {:else if n.code}
{/if} {:else if tab === 'drift' && n.sql} - goto(`/catalog/${m}?tab=drift`)} /> + goto(to(`/catalog/${m}?tab=drift`))} /> {:else if tab === 'lineage'} {#if graph.nodes.length} diff --git a/ui/src/routes/catalog/[name]/overview-tab.svelte b/ui/src/routes/catalog/[name]/overview-tab.svelte index 63e874ca49..2a63d18439 100644 --- a/ui/src/routes/catalog/[name]/overview-tab.svelte +++ b/ui/src/routes/catalog/[name]/overview-tab.svelte @@ -1,4 +1,5 @@ -
+
{#if connections.error}

Couldn't load connections: {connections.error}

{/if} - - {#each warehouse as c (c.name)} -
-
+ {#each list as c (c.id)} + +
+
- {c.name} - {c.adapter} · {c.purpose === 'catalog' ? 'used to read SQLBuild’s own tables back (warehouse sync)' : `builds target ${c.target}`} · {c.sourceFile} + Warehouse +
connections.refresh()} />
-
- {#each c.settings as f (f.key)} -
-
{f.key}
-
{shownValue(c, f)}
-
- {/each} -
- + {#if c.tables.length} + + + + {#each c.tables as t (t.table)} + + + + + + + + {/each} + +
Synced tableNewest syncRecordsTookLast clean sync
{t.table} + + {#if t.failure}{:else}{/if} + + {formatNumber(t.recordsSent)}{formatMs(t.elapsedMs)}{t.lastSyncedAt ? formatAgo(t.lastSyncedAt, app.now) : 'never'}
+ {:else} +

No warehouse sync reported yet.

+ {/if}
- {:else} -

{connections.data ? 'This location’s target has no warehouse connection. Add one to the target in the project config.' : connections.error ? '' : 'Loading…'}

- {/each} - - + + {:else} +

{connections.data ? 'No location you can see in full has reported yet.' : connections.slow ? 'Loading…' : ''}

+ {/each}
diff --git a/ui/src/routes/connections/[id]/+page.svelte b/ui/src/routes/connections/[id]/+page.svelte deleted file mode 100644 index 4c73c23cf7..0000000000 --- a/ui/src/routes/connections/[id]/+page.svelte +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - -
-
- ← Connections - {name} - {#if warehouse} - Warehouse · {warehouse.adapter} · target {warehouse.target} - {/if} -
- - {#if connections.error} -
Couldn't load connections: {connections.error}
- {:else if connections.data && !warehouse} -
- {app.location?.name} has no warehouse connection named {name}. Looking for an API or storage connection? See Providers. -
- {:else if warehouse} -
-
-
-
- -
-
- - Read-only. Set in {warehouse.sourceFile}; secrets show only where they come from. -
- -
-
- - -
- {:else} -
Loading…
- {/if} -
diff --git a/ui/src/routes/jobs/+page.svelte b/ui/src/routes/jobs/+page.svelte index 5f1243506b..f8c8024d2b 100644 --- a/ui/src/routes/jobs/+page.svelte +++ b/ui/src/routes/jobs/+page.svelte @@ -1,4 +1,5 @@ - {#if picked.length && canRunPicked} + {#if picked.length && def?.kind === 'job'} - goto(planHref(pickedNodes.map((n) => n.key)))}>▷ Run {picked.length} selected + goto(to(planHref([], 'build', { job: name, exclude: picked })))}>▷ Run all but {picked.length} + goto(to(planHref(picked, 'build', { job: name })))}>▷ Run {picked.length} selected + {:else if picked.length && canRunPicked} + + goto(to(planHref(pickedNodes.map((n) => n.key))))}>▷ Run {picked.length} selected {:else}
- ← Jobs + ← Jobs {name} {#if def} {def.kind === 'group' ? `Group${def.members ? ` · ${def.members.length} jobs` : ''}` : 'Job'} · {def.nodeCount} nodes {#each jobTriggers as t (t.id)} - {t.cadence} + {t.cadence} {/each} {#if latest} - + Latest run: {runStatusLabel[latest.status]} · {formatAgo(latest.startedAt ?? latest.queuedAt, app.now)} {/if} {#if def.criticalNodes.length} - Critical: {#each def.criticalNodes as n, i (n)}{i ? ', ' : ''}{n}{/each} + Critical: {#each def.criticalNodes as n, i (n)}{i ? ', ' : ''}{n}{/each} {/if} {/if} @@ -144,7 +149,7 @@ {@const n = pickedNodes[0]}

{kindLine(n)}

{n.key}

- Open in Catalog ↗ + Open in Catalog ↗ {#if n.description}

{n.description}

{/if}
{#if n.lastOutcome || n.lastBuild}
Last build
{n.lastOutcome ? nodeOutcomeLabel[n.lastOutcome] : 'Built'}{n.lastBuild ? ` · ${formatAgo(n.lastBuild.builtAt, app.now)}` : ''}
{/if} @@ -155,7 +160,7 @@ {:else if pickedNodes.length > 1}

{pickedNodes.length} nodes selected

    - {#each pickedNodes as n (n.key)}
  • {n.key}
    • {/each} + {#each pickedNodes as n (n.key)}
  • {n.key}
    • {/each}
{:else} {#if def.description}

Description

{def.description}

{/if} @@ -165,11 +170,11 @@ {#if def.overlap}
If already running
{overlapPlain[def.overlap]}
{/if} {#if def.onFailure}
If something fails
{onFailurePlain[def.onFailure]}
{/if} {#if def.priority !== null}
Priority
{def.priority}
{/if} - {#if def.declaration}
Targets
{def.targets?.join(', ') ?? 'all'}
{/if} + {#if def.targets}
Targets
{def.targets.join(', ')}
{/if} {#if def.owner}
Owner
{def.owner}
{/if} {#if def.sourceFile}
Defined in
{def.sourceFile}
{/if} {#if def.memberOf?.length} -
In groups
{#each def.memberOf as g, i (g)}{i ? ', ' : ''}{g}{/each}
+
In groups
{#each def.memberOf as g, i (g)}{i ? ', ' : ''}{g}{/each}
{/if}
{#if def.settings && Object.keys(def.settings).length} @@ -183,7 +188,7 @@ {#each def.concurrency as g (g.key)}

{g.holders.length} of {g.limit} {slotLabel(g.key)} in use - {#each g.holders as h (h)}{shortRunId(h)}{/each} + {#each g.holders as h (h)}{shortRunId(h)}{/each}

{/each} {:else if def.concurrency === null && def.concurrencyGroups.length} @@ -202,9 +207,9 @@ {#each runs as r (r.id)} - goto(`/runs/${r.id}`)}> + goto(to(`/runs/${r.id}`))}> {runStatusLabel[r.status]} - {shortRunId(r.id)} + {shortRunId(r.id)}{#if r.narrowing}part{/if} {runOriginLabel[r.origin]}{r.parentRunId ? ` in group run ${shortRunId(r.parentRunId)}` : r.triggerName ? ` · ${r.triggerName}` : ''} {formatNodeCounts(r.nodeCounts)} {r.startedAt ? formatDuration(r.startedAt, r.endedAt, app.now) : '—'} @@ -263,6 +268,11 @@ } .chip i, dd i, + .runs td .part { + margin-left: 8px; + color: var(--muted-foreground); + font-size: 11.5px; + } .runs td i { display: inline-block; width: 7px; diff --git a/ui/src/routes/jobs/[name]/launch-form.svelte b/ui/src/routes/jobs/[name]/launch-form.svelte index af7a63c91b..6805e942fe 100644 --- a/ui/src/routes/jobs/[name]/launch-form.svelte +++ b/ui/src/routes/jobs/[name]/launch-form.svelte @@ -1,4 +1,5 @@