feat(orchestration)!: orchestration names join the global unique-name check (#1394)

* feat(orchestration): jobs, groups, sensors and calendars join the global unique-name check

Publication now rejects a job, job group, sensor or calendar schedule whose
name is already used by any other project resource or orchestration
declaration, with the same D007 conflict discovery raises for models,
sources and the rest. The narrower duplicate-job and duplicate-trigger
checks are removed; the object-identity check stays.

build_orchestration_snapshot and compile_snapshot take the discovered
project inputs so publication can build the full name set.

Fixtures whose jobs reused model names are renamed, and DuckShop gains a
name-collision edge case.

* docs: sync the docs skill with the python nodes page

* feat(orchestration): say how to rename a colliding declaration

A name conflict already names both declarations and their files. It now
ends with how to resolve it, and when a job, job group, sensor or calendar
schedule is involved, says where that name comes from: a job, group or
sensor takes it from its first argument, a calendar schedule from the
function it decorates.

* refactor(orchestration): collect declared names before scanning for conflicts

The name check now gathers every declaration as a named record and then
scans them once, instead of threading a mutable map through a helper.
The first conflict found is the same.

Author: kvlonge

benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py

@@ -2,7 +2,7 @@
 
 from sqlbuild.orchestration import calendar_schedule, job
 
-orders = job("orders", description="Build order reporting.", select="orders")
+orders = job("order_reporting", description="Build order reporting.", select="orders")
 
 
 @calendar_schedule(

benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py

@@ -3,7 +3,7 @@
 from sqlbuild.orchestration import job
 
 orders = job(
-    "orders",
+    "refresh_orders",
     description="Refresh orders.",
     select="orders",
     settings={"load_sources": False, "reload": True},

benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py

@@ -3,7 +3,7 @@
 from sqlbuild.orchestration import job
 
 orders = job(
-    "orders",
+    "refresh_orders",
     description="Refresh orders.",
     select="orders",
     settings={"select": "customers"},

benchmarks/duckshop/edge_cases/orchestration/name_collision.py

@@ -0,0 +1,5 @@
+"""A job may not reuse a model's name: jobs share the project's one resource namespace."""
+
+from sqlbuild.orchestration import job
+
+orders = job("orders", description="Build orders.", select="orders")

benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json

@@ -1,9 +1,9 @@
 {
   "description": "Three minutely cron jobs start ten hours back with the default latest catch-up. Two installed schedulers catch up while being SIGKILLed at seeded random moments; every lapse of a killed leader's lease is simulated. After a final uninterrupted scheduler, each trigger has ticked its latest due minute exactly once and nothing off schedule, each tick requests or is covered by exactly one run, and every requested run has one request event.",
   "jobs": [
-    "orders",
-    "customers",
-    "inventory"
+    "refresh_orders",
+    "refresh_customers",
+    "refresh_inventory"
   ],
   "backlog_minutes": 600,
   "rounds": 16,

crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs

@@ -1,73 +1,149 @@
-//! Python's `_validate_unique_project_resource_names`: one namespace across resource kinds.
+//! One namespace across resource kinds, orchestration declarations included.
 
 use std::collections::{HashMap, HashSet};
 
 use crate::project_validation::_helpers::conflicts::{conflict, schema_seeds};
 use crate::project_validation::_helpers::identities::block_or_file_name;
-use crate::project_validation::models::{NodeKind, ProjectFacts, ValidationFailure};
+use crate::project_validation::constants::ORCHESTRATION_RENAME_ADVICE;
+use crate::project_validation::models::{
+    DeclaredName, NamedPath, NodeKind, ProjectFacts, ValidationFailure,
+};
 
 /// Reject the first resource name declared twice across models, sources, seeds and the rest.
 pub(crate) fn validate_unique_project_resource_names(
     facts: &ProjectFacts,
 ) -> Result<(), ValidationFailure> {
-    let mut seen: HashMap<String, (String, String)> = HashMap::new();
-    let mut declare = |name: &str, kind: &str, path: &str| -> Result<(), ValidationFailure> {
-        if let Some((existing_kind, existing_path)) = seen.get(name) {
-            return Err(conflict(format!(
-                "Project resource name '{name}' is declared as both {existing_kind} in \
-                 {existing_path} and {kind} in {path}; model, source, seed, function, loader, \
-                 task, asset, check, provider, hook, unit test, and scenario names must be \
-                 globally unique"
-            )));
+    let seeds: Vec<NamedPath> = schema_seeds(facts);
+    let mut seen: HashMap<&str, DeclaredName<'_>> = HashMap::new();
+    for declared in declarations(facts, &seeds) {
+        if let Some(first) = seen.get(declared.name) {
+            return Err(name_conflict(first, &declared));
         }
-        seen.insert(name.to_owned(), (kind.to_owned(), path.to_owned()));
-        Ok(())
-    };
+        seen.insert(declared.name, declared);
+    }
+    Ok(())
+}
+
+/// Every declared name in the order conflicts are reported.
+fn declarations<'a>(facts: &'a ProjectFacts, seeds: &'a [NamedPath]) -> Vec<DeclaredName<'a>> {
+    let mut declared: Vec<DeclaredName<'a>> = Vec::new();
     for model in &facts.models {
-        declare(&model.file.name, "model", &model.file.path)?;
+        declared.push(DeclaredName {
+            name: &model.file.name,
+            kind: "model",
+            path: &model.file.path,
+            orchestration: false,
+        });
     }
     let mut managed_sources: HashSet<&str> = HashSet::new();
     for file in &facts.sources {
         for entry in &file.entries {
             if entry.managed {
                 managed_sources.insert(&entry.name);
             }
-            declare(&entry.name, "source", &file.path)?;
+            declared.push(DeclaredName {
+                name: &entry.name,
+                kind: "source",
+                path: &file.path,
+                orchestration: false,
+            });
         }
     }
-    for seed in schema_seeds(facts) {
-        declare(&seed.name, "seed", &seed.path)?;
+    for seed in seeds {
+        declared.push(DeclaredName {
+            name: &seed.name,
+            kind: "seed",
+            path: &seed.path,
+            orchestration: false,
+        });
     }
     for function in facts.sql_functions.iter().chain(&facts.python_functions) {
-        declare(&function.name, "function", &function.path)?;
+        declared.push(DeclaredName {
+            name: &function.name,
+            kind: "function",
+            path: &function.path,
+            orchestration: false,
+        });
     }
     for node in &facts.nodes {
         if node.kind == NodeKind::Loader && managed_sources.contains(node.name.as_str()) {
             continue;
         }
-        declare(&node.name, node.kind.label(), &node.path)?;
+        declared.push(DeclaredName {
+            name: &node.name,
+            kind: node.kind.label(),
+            path: &node.path,
+            orchestration: false,
+        });
     }
     for provider in &facts.providers {
-        declare(&provider.name, "provider", &provider.path)?;
+        declared.push(DeclaredName {
+            name: &provider.name,
+            kind: "provider",
+            path: &provider.path,
+            orchestration: false,
+        });
     }
     for hook in &facts.sql_hooks {
-        declare(&hook.name, "sql hook", &hook.path)?;
+        declared.push(DeclaredName {
+            name: &hook.name,
+            kind: "sql hook",
+            path: &hook.path,
+            orchestration: false,
+        });
     }
     for hook in &facts.hook_functions {
-        declare(&hook.name, "python hook", &hook.path)?;
+        declared.push(DeclaredName {
+            name: &hook.name,
+            kind: "python hook",
+            path: &hook.path,
+            orchestration: false,
+        });
     }
     for scenario in &facts.scenarios {
-        declare(&scenario.name, "scenario", &scenario.path)?;
+        declared.push(DeclaredName {
+            name: &scenario.name,
+            kind: "scenario",
+            path: &scenario.path,
+            orchestration: false,
+        });
     }
     let mut test_names: HashSet<&str> = HashSet::new();
     for test in &facts.tests {
         for block in &test.blocks {
             let name: &str = block_or_file_name(block.name.as_deref(), &test.file.name);
-            if !test_names.insert(name) {
-                continue;
+            if test_names.insert(name) {
+                declared.push(DeclaredName {
+                    name,
+                    kind: "unit test",
+                    path: &test.file.path,
+                    orchestration: false,
+                });
             }
-            declare(name, "unit test", &test.file.path)?;
         }
     }
-    Ok(())
+    for declaration in &facts.orchestration {
+        declared.push(DeclaredName {
+            name: &declaration.name,
+            kind: declaration.kind,
+            path: &declaration.path,
+            orchestration: true,
+        });
+    }
+    declared
+}
+
+/// Name both declarations and say how to rename one of them.
+fn name_conflict(first: &DeclaredName<'_>, second: &DeclaredName<'_>) -> ValidationFailure {
+    let advice: &str = if first.orchestration || second.orchestration {
+        ORCHESTRATION_RENAME_ADVICE
+    } else {
+        ""
+    };
+    conflict(format!(
+        "Project resource name '{}' is declared as both {} in {} and {} in {}; model, source, seed, function, loader, task, asset, check, provider, \
+         hook, unit test, scenario, job, job group, sensor, and calendar schedule names must be \
+         globally unique; rename one of them{advice}",
+        first.name, first.kind, first.path, second.kind, second.path
+    ))
 }

crates/sqlbuild-discovery/src/project_validation/constants.rs

@@ -12,3 +12,7 @@ pub(crate) const RESERVED_MODEL_NAMES: [&str; 1] = ["_chain_"];
 pub(crate) const INITIAL_SEED_READ_BYTES: u64 = 64 * 1024;
 /// A header and two data rows: enough to know a seed holds more than one data row.
 pub(crate) const RECORDS_FOR_MANY_ROWS: usize = 3;
+
+/// How a colliding orchestration declaration is renamed, added to its name conflict.
+pub(crate) const ORCHESTRATION_RENAME_ADVICE: &str = " (a job, job group or sensor takes its name \
+     from its first argument, and a calendar schedule from the function it decorates)";

crates/sqlbuild-discovery/src/project_validation/main/mod.rs

@@ -1,2 +1,3 @@
 pub mod at_most_one_csv_data_row;
+pub mod validate_orchestration_names;
 pub mod validate_project_inputs;

crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs

@@ -0,0 +1,9 @@
+//! Orchestration declarations share the project's one resource namespace.
+
+use crate::project_validation::_helpers::resource_names::validate_unique_project_resource_names;
+use crate::project_validation::models::{ProjectFacts, ValidationFailure};
+
+/// Reject the first name a job, group, sensor or calendar schedule shares with anything else.
+pub fn validate_orchestration_names(facts: &ProjectFacts) -> Result<(), ValidationFailure> {
+    validate_unique_project_resource_names(facts)
+}

crates/sqlbuild-discovery/src/project_validation/models.rs

@@ -178,6 +178,26 @@ pub struct ProjectFacts {
     pub project_model_paths: Option<Vec<String>>,
     /// Python's `csv.field_size_limit()`, or `None` for its default.
     pub csv_field_limit: Option<usize>,
+    /// Jobs, job groups, sensors and calendar schedules, known only when a location publishes.
+    pub orchestration: Vec<OrchestrationNameFact>,
+}
+
+/// One name in the project's resource namespace, as a name conflict reports it.
+#[derive(Debug)]
+pub(crate) struct DeclaredName<'a> {
+    pub(crate) name: &'a str,
+    pub(crate) kind: &'a str,
+    pub(crate) path: &'a str,
+    /// A job, job group, sensor or calendar schedule, whose conflicts say how it is named.
+    pub(crate) orchestration: bool,
+}
+
+/// A named orchestration declaration: its name, its kind as conflicts word it, and its module.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct OrchestrationNameFact {
+    pub name: String,
+    pub kind: &'static str,
+    pub path: String,
 }
 
 /// Why discovered inputs are rejected: a discovery error, or a seed CSV Python cannot read.