feat(orchestration): real plans, partial job runs, queue and timeline views and trigger details in the UI

crates/sqlbuild-analysis/src/dag_artifact/_helpers/selection.rs

@@ -9,8 +9,8 @@ use sqlbuild_core::text::main::python_isspace::python_isspace;
 use sqlbuild_core::text::main::python_str_repr::python_str_repr;
 
 use crate::dag_artifact::constants::{
-    SELECTOR_KIND_CHECK, SELECTOR_KIND_NAME, SELECTOR_KIND_PATH, SELECTOR_KIND_SEPARATOR,
-    SELECTOR_KIND_TAG, SELECTOR_KIND_TEST, SELECTOR_MODELS_PREFIX,
+    SELECTOR_KIND_CHECK, SELECTOR_KIND_JOB, SELECTOR_KIND_NAME, SELECTOR_KIND_PATH,
+    SELECTOR_KIND_SEPARATOR, SELECTOR_KIND_TAG, SELECTOR_KIND_TEST, SELECTOR_MODELS_PREFIX,
 };
 use crate::dag_artifact::errors::DagSelectionError;
 use crate::dag_artifact::models::{DagSelectionIndex, DagSelectionNode};
@@ -146,6 +146,12 @@ fn match_kind(
                 "{SELECTOR_KIND_TEST}{SELECTOR_KIND_SEPARATOR}{value}"
             )));
         }
+        SELECTOR_KIND_JOB => {
+            return Err(DagSelectionError::Input(format!(
+                "selector '{SELECTOR_KIND_JOB}{SELECTOR_KIND_SEPARATOR}{value}' selects a job; \
+                 select the job's assets by name, tag or path instead"
+            )));
+        }
         SELECTOR_KIND_TAG => ids(&|node| node.tags.iter().any(|tag| tag == value)),
         SELECTOR_KIND_PATH => {
             let folder: String = value.replace('\\', "/");

crates/sqlbuild-analysis/src/dag_artifact/constants.rs

@@ -32,6 +32,7 @@ pub(crate) const SELECTOR_KIND_TAG: &str = "tag";
 pub(crate) const SELECTOR_KIND_PATH: &str = "path";
 pub(crate) const SELECTOR_KIND_CHECK: &str = "check";
 pub(crate) const SELECTOR_KIND_TEST: &str = "test";
+pub(crate) const SELECTOR_KIND_JOB: &str = "job";
 pub(crate) const SELECTOR_KIND_SEPARATOR: &str = ":";
 pub(crate) const SELECTOR_MODELS_PREFIX: &str = "models/";
 /// The orchestrator kind every SQLBuild asset carries.

crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs

@@ -0,0 +1,51 @@
+//! Jobs and job groups resolved once on the graph, so `job:` selects what the job publishes.
+
+use std::collections::BTreeMap;
+
+use crate::graph::_helpers::selectors::{Keys, resolve};
+use crate::graph::constants::UNKNOWN_NAME_CODE;
+use crate::graph::errors::SelectorError;
+use crate::graph::models::{JobSelection, ProjectGraph};
+
+/// Resolve every job on the graph without jobs attached, as publication does, then attach them.
+pub(crate) fn attach(
+    mut graph: ProjectGraph,
+    catalog: BTreeMap<String, JobSelection>,
+) -> ProjectGraph {
+    graph.jobs = None;
+    let mut resolved: BTreeMap<String, Result<Keys, SelectorError>> = BTreeMap::new();
+    for (name, selection) in &catalog {
+        let keys: Result<Keys, SelectorError> = match selection {
+            JobSelection::Job { select, exclude } => {
+                resolve(&graph, select, exclude).map(|keys| keys.into_iter().collect())
+            }
+            JobSelection::Resolved { keys } => Ok(keys.iter().cloned().collect()),
+            JobSelection::Group { .. } => continue,
+        };
+        resolved.insert(name.clone(), keys);
+    }
+    for (name, selection) in &catalog {
+        if let JobSelection::Group { members } = selection {
+            let mut keys: Keys = Keys::new();
+            let mut failure: Option<SelectorError> = None;
+            for member in members {
+                match resolved.get(member) {
+                    Some(Ok(member_keys)) => keys.extend(member_keys.iter().cloned()),
+                    Some(Err(error)) => failure = failure.or_else(|| Some(error.clone())),
+                    None => {
+                        failure = failure.or_else(|| {
+                            Some(SelectorError {
+                                code: UNKNOWN_NAME_CODE,
+                                message: format!("job group '{name}' names unknown job '{member}'"),
+                                help: None,
+                            })
+                        });
+                    }
+                }
+            }
+            resolved.insert(name.clone(), failure.map_or(Ok(keys), Err));
+        }
+    }
+    graph.jobs = Some(resolved);
+    graph
+}

crates/sqlbuild-analysis/src/graph/_helpers/mod.rs

@@ -1,5 +1,6 @@
 pub(crate) mod closure;
 pub(crate) mod indexes;
+pub(crate) mod jobs;
 pub(crate) mod layers;
 pub(crate) mod selectors;
 pub(crate) mod unique_ids;

crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs

@@ -5,7 +5,7 @@ use std::collections::BTreeSet;
 use crate::assembly::project::types::ObjectKey;
 use crate::graph::constants::{
     EMPTY_SELECTOR_CODE, EMPTY_VALUE_CODE, EXPANSION_MARKER, FOLDER_SEPARATOR,
-    INTERSECTION_SEPARATOR, KIND_SEPARATOR, MISPLACED_MARKER_CODE, MISSING_NAME_CODE,
+    INTERSECTION_SEPARATOR, JOB_KIND, KIND_SEPARATOR, MISPLACED_MARKER_CODE, MISSING_NAME_CODE,
     MODEL_RESOURCE, MODEL_ROOT, MODEL_ROOT_PREFIX, NAME_KIND, PATH_KIND, PATH_ROOT_CODE,
     PATH_ROOT_ERROR, PATH_SELECTOR_CODE, PATH_SEPARATOR, PATTERN_CHARACTERS, PLANNER_DEFAULT_CODE,
     SEED_RESOURCE, SOURCE_RESOURCE, SQL_FILE_SUFFIX, SUGGESTION_CUTOFF, SUGGESTION_LIMIT,
@@ -161,10 +161,41 @@ fn path(graph: &ProjectGraph, names: (&str, &str), expand: (bool, bool)) -> Reso
 }
 
 fn named(graph: &ProjectGraph, name: &str) -> Result<ObjectKey, SelectorError> {
-    graph
-        .key_named(name)
-        .cloned()
-        .ok_or_else(|| error(UNKNOWN_NAME_CODE, format!("unknown selector name '{name}'")))
+    if let Some(key) = graph.key_named(name) {
+        return Ok(key.clone());
+    }
+    let job: bool = name.starts_with(&format!("{JOB_KIND}{KIND_SEPARATOR}"))
+        || graph
+            .jobs
+            .as_ref()
+            .is_some_and(|jobs| jobs.contains_key(name));
+    Err(if job {
+        error(
+            PATH_SELECTOR_CODE,
+            format!("path selector endpoint '{name}' is a job; path selectors join two nodes"),
+        )
+    } else {
+        error(UNKNOWN_NAME_CODE, format!("unknown selector name '{name}'"))
+    })
+}
+
+/// Whether a `job:` part or an unknown bare name in `selectors` needs jobs; paths never do.
+pub(crate) fn needs_jobs(graph: &ProjectGraph, selectors: &[String], also_known: &[&str]) -> bool {
+    selectors
+        .iter()
+        .flat_map(|selector| selector.split(char::is_whitespace))
+        .flat_map(|token| token.split(INTERSECTION_SEPARATOR))
+        .filter(|part| !part.is_empty())
+        .any(|part| match parse(part) {
+            Ok(ParsedSelector::Kind { kind, value, .. }) => {
+                kind == JOB_KIND
+                    || (kind == NAME_KIND
+                        && !is_pattern(&value)
+                        && graph.key_named(&value).is_none()
+                        && !also_known.contains(&value.as_str()))
+            }
+            _ => false,
+        })
 }
 
 pub(crate) fn parse(raw: &str) -> Result<ParsedSelector, SelectorError> {
@@ -241,6 +272,7 @@ fn is_selector_kind(prefix: &str) -> bool {
         TAG_KIND,
         PATH_KIND,
         TEST_KIND,
+        JOB_KIND,
     ]
     .contains(&prefix)
         || UNMAPPED_KINDS.contains(&prefix)
@@ -260,6 +292,7 @@ pub(crate) fn matched(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved
             Ok(keys)
         }
         PATH_KIND => folder(graph, value),
+        JOB_KIND => job(graph, value),
         TEST_KIND => Err(error(
             UNIT_TEST_CODE,
             format!(
@@ -273,7 +306,17 @@ pub(crate) fn matched(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved
                 return Ok(keys);
             }
             let pattern: bool = is_pattern(value);
-            let names: Vec<&str> = graph.names.iter().map(|(name, _)| name.as_str()).collect();
+            let names: Vec<&str> = graph
+                .names
+                .iter()
+                .map(|(name, _)| name.as_str())
+                .chain(
+                    graph
+                        .jobs
+                        .iter()
+                        .flat_map(|jobs| jobs.keys().map(String::as_str)),
+                )
+                .collect();
             Err(SelectorError {
                 code: UNKNOWN_NAME_CODE,
                 message: format!(
@@ -309,6 +352,12 @@ fn lookup(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved {
     };
     let kind_matches = |key: &ObjectKey| resource.is_none_or(|resource| key.0 == resource);
     if !is_pattern(value) {
+        if resource.is_none()
+            && graph.key_named(value).is_none()
+            && let Some(Some(keys)) = graph.jobs.as_ref().map(|jobs| jobs.get(value))
+        {
+            return keys.clone();
+        }
         return Ok(graph
             .key_named(value)
             .filter(|key| kind_matches(key))
@@ -324,6 +373,40 @@ fn lookup(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved {
         .collect())
 }
 
+/// The nodes the jobs and job groups `value` names or matches select.
+fn job(graph: &ProjectGraph, value: &str) -> Resolved {
+    let Some(jobs) = &graph.jobs else {
+        return Err(error(
+            UNMAPPED_KIND_CODE,
+            format!(
+                "selector '{JOB_KIND}{KIND_SEPARATOR}{value}' selects a job; jobs cannot be selected here"
+            ),
+        ));
+    };
+    let pattern: bool = is_pattern(value);
+    let mut keys: Keys = Keys::new();
+    let mut found: bool = false;
+    for (name, selected) in jobs {
+        if name == value || (pattern && glob_matches(value, name)) {
+            found = true;
+            keys.extend(selected.clone()?);
+        }
+    }
+    if found {
+        return Ok(keys);
+    }
+    Err(SelectorError {
+        code: UNKNOWN_NAME_CODE,
+        message: format!(
+            "unknown job {} '{value}'",
+            if pattern { "pattern" } else { "name" }
+        ),
+        help: (!pattern)
+            .then(|| suggestion(value, jobs.keys().map(String::as_str).collect()))
+            .flatten(),
+    })
+}
+
 fn folder(graph: &ProjectGraph, value: &str) -> Resolved {
     let folder: String = value.replace('\\', "/").trim_matches('/').to_owned();
     let selector: &str = if folder == MODEL_ROOT {

crates/sqlbuild-analysis/src/graph/constants.rs

@@ -18,6 +18,7 @@ pub(crate) const SQL_FILE_SUFFIX: &str = ".sql";
 pub(crate) const TAG_KIND: &str = "tag";
 pub(crate) const PATH_KIND: &str = "path";
 pub(crate) const TEST_KIND: &str = "test";
+pub(crate) const JOB_KIND: &str = "job";
 /// Selector kinds Python parses but `sqb compile` cannot map to a compiled resource.
 pub(crate) const UNMAPPED_KINDS: [&str; 4] = ["task", "asset", "loader", "check"];
 pub(crate) const SUGGESTION_CUTOFF: f64 = 0.8;

crates/sqlbuild-analysis/src/graph/main/attach_jobs.rs

@@ -0,0 +1,8 @@
+use std::collections::BTreeMap;
+
+use crate::graph::models::{JobSelection, ProjectGraph};
+
+/// Attach jobs and job groups so `job:<name>` and a bare job name select what each job selects.
+pub fn attach_jobs(graph: ProjectGraph, catalog: BTreeMap<String, JobSelection>) -> ProjectGraph {
+    crate::graph::_helpers::jobs::attach(graph, catalog)
+}

crates/sqlbuild-analysis/src/graph/main/mod.rs

@@ -1,3 +1,4 @@
+pub mod attach_jobs;
 pub mod build_project_graph;
 pub mod build_resources;
 pub mod downstream_edges;
@@ -11,4 +12,5 @@ pub mod reachable_keys;
 pub mod resolve_selector_tokens;
 pub mod resolve_selectors;
 pub mod selector_name_help;
+pub mod selectors_need_jobs;
 pub mod unique_id_edges;

crates/sqlbuild-analysis/src/graph/main/selectors_need_jobs.rs

@@ -0,0 +1,10 @@
+use crate::graph::models::ProjectGraph;
+
+/// Whether `selectors` name a job, either by `job:` or as a bare name the graph and `also_known` lack.
+pub fn selectors_need_jobs(
+    graph: &ProjectGraph,
+    selectors: &[String],
+    also_known: &[&str],
+) -> bool {
+    crate::graph::_helpers::selectors::needs_jobs(graph, selectors, also_known)
+}

crates/sqlbuild-analysis/src/graph/models.rs

@@ -1,8 +1,9 @@
 //! The project graph, the resources it is built from and selector failures.
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 
 use crate::assembly::project::types::ObjectKey;
+use crate::graph::errors::SelectorError;
 
 /// Dependency lists indexed without losing project insertion order.
 #[derive(Debug, Default)]
@@ -37,6 +38,24 @@ pub struct ProjectGraph {
     pub(crate) upstream_index: HashMap<ObjectKey, usize>,
     pub(crate) downstream_index: HashMap<ObjectKey, usize>,
     pub(crate) name_index: HashMap<String, usize>,
+    /// Each job and job group's resolved selection, or why it fails; `None` until a command attaches jobs.
+    pub jobs: Option<BTreeMap<String, Result<BTreeSet<ObjectKey>, SelectorError>>>,
+}
+
+/// What one job or job group selects: its own selector tokens, its member jobs, or resolved keys.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum JobSelection {
+    Job {
+        select: Vec<String>,
+        exclude: Vec<String>,
+    },
+    Group {
+        members: Vec<String>,
+    },
+    /// A selection already resolved, such as a published job's.
+    Resolved {
+        keys: Vec<ObjectKey>,
+    },
 }
 
 /// One parsed selector token: `kind:value` (kind `name` when bare) or `start~end`.

crates/sqlbuild-analysis/src/graph/tests.rs

@@ -2,6 +2,8 @@
 mod graph;
 #[path = "tests/helpers.rs"]
 mod helpers;
+#[path = "tests/test_job_selectors.rs"]
+mod job_selectors;
 #[path = "tests/test_selector_scenarios.rs"]
 mod selector_scenarios;
 #[path = "tests/test_types.rs"]

crates/sqlbuild-analysis/src/graph/tests/helpers.rs

@@ -1,10 +1,12 @@
-use std::collections::BTreeSet;
+use std::collections::{BTreeMap, BTreeSet};
 
 use crate::assembly::project::types::ObjectKey;
 use crate::graph::errors::SelectorError;
+use crate::graph::main::attach_jobs::attach_jobs;
 use crate::graph::main::build_project_graph::build_project_graph;
 use crate::graph::main::resolve_selectors::resolve_selectors;
-use crate::graph::models::{GraphResource, ParsedSelector, ProjectGraph};
+use crate::graph::main::selectors_need_jobs::selectors_need_jobs;
+use crate::graph::models::{GraphResource, JobSelection, ParsedSelector, ProjectGraph};
 use crate::graph::tests::test_types::SelectorFailure;
 
 type OwnedFailure = (String, String, Option<String>);
@@ -225,3 +227,48 @@ pub(super) fn path(start: &str, end: &str, upstream: bool, downstream: bool) ->
         downstream,
     }
 }
+
+/// Jobs over the orders project: valid jobs, a group, and jobs whose own selection fails.
+fn orders_jobs() -> BTreeMap<String, JobSelection> {
+    let job = |select: &[&str], exclude: &[&str]| JobSelection::Job {
+        select: select.iter().map(|value| (*value).to_owned()).collect(),
+        exclude: exclude.iter().map(|value| (*value).to_owned()).collect(),
+    };
+    let group = |members: &[&str]| JobSelection::Group {
+        members: members.iter().map(|value| (*value).to_owned()).collect(),
+    };
+    BTreeMap::from([
+        ("nightly".to_owned(), job(&["orders"], &[])),
+        ("crm".to_owned(), job(&["customers"], &[])),
+        ("marts".to_owned(), job(&["orders+"], &[])),
+        (
+            "marts_without_items".to_owned(),
+            job(&["orders+"], &["order_items"]),
+        ),
+        ("daily_group".to_owned(), group(&["nightly", "crm"])),
+        ("broken".to_owned(), job(&["missing"], &[])),
+        ("nested".to_owned(), job(&["job:nightly"], &[])),
+        ("ghost_group".to_owned(), group(&["nightly", "ghost"])),
+    ])
+}
+
+/// The names `select` minus `exclude` resolves to in the orders project, with or without its jobs.
+pub(super) fn job_selection(
+    attached: bool,
+    select: &[&str],
+    exclude: &[&str],
+) -> Result<BTreeSet<String>, OwnedFailure> {
+    let mut graph: ProjectGraph = attach_jobs(
+        orders_graph(),
+        orders_jobs().into_iter().filter(|_| attached).collect(),
+    );
+    graph.jobs = graph.jobs.filter(|_| attached);
+    resolved_in(&graph, select, exclude)
+        .map(|keys| keys.into_iter().map(|(_, name)| name).collect())
+}
+
+/// Whether `selectors` need jobs on the orders project, `also_known` naming other node names.
+pub(super) fn needs_jobs(selectors: &[&str], also_known: &[&str]) -> bool {
+    let selectors: Vec<String> = selectors.iter().map(|value| (*value).to_owned()).collect();
+    selectors_need_jobs(&orders_graph(), &selectors, also_known)
+}