From 99fd4b83e7f1ed7d29ea58e8cb015fb8769d9900 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 5 Oct 2026 11:54:44 +0200 Subject: [PATCH 01/12] Add job_runs lifecycle.triggers.on_value_change Adds `lifecycle.triggers.on_value_change` for `resources.job_runs` (direct engine): watch an interpolated expression (a variable, workspace field, or a ${resources.*} reference) and re-fire the run when its resolved value changes. - Snapshot the expression before interpolation and use the trimmed form as the stable state key (SnapshotJobRunValueTriggers), so a value change is a modification of one entry rather than a change to the set of watches. - Resolution reuses existing mechanisms: normal variable interpolation resolves ${var.*}/${workspace.*} into triggers_state, and the planner resolves ${resources.*} via the references PrepareInputConfig registers, so the deploy graph depends on them. The resolved value is stored in state directly. - Removing or adding a watch is classified in OverrideChangeDesc alongside on_file_change: adding/changing recreates the run, removing keeps it. Built on the on_file_change work (#6309): no PersistState, path.String()-based change classification, prevent_destroy left to the generic checkForPreventDestroy. Co-authored-by: Isaac --- .../bundles/job-runs-on-value-change.md | 1 + .../job_runs/on_value_change/databricks.yml | 33 +++ .../job_runs/on_value_change/out.test.toml | 2 + .../job_runs/on_value_change/output.txt | 239 ++++++++++++++++++ .../resources/job_runs/on_value_change/script | 87 +++++++ .../job_runs/on_value_change/test.toml | 9 + .../job_runs/on_value_change/watched.txt | 1 + .../on_value_change_errors/databricks.yml | 21 ++ .../on_value_change_errors/out.test.toml | 2 + .../on_value_change_errors/output.txt | 51 ++++ .../job_runs/on_value_change_errors/script | 14 + .../job_runs/on_value_change_errors/test.toml | 1 + .../on_value_change_errors/watched.txt | 1 + .../on_value_change_ref/databricks.yml | 37 +++ .../on_value_change_ref/out.test.toml | 3 + .../job_runs/on_value_change_ref/output.txt | 107 ++++++++ .../job_runs/on_value_change_ref/script | 38 +++ .../job_runs/on_value_change_ref/test.toml | 11 + .../mutator/resolve_job_run_file_triggers.go | 5 +- .../process_static_resources.go | 3 + .../resourcemutator/resource_mutator.go | 3 + .../snapshot_job_run_value_triggers.go | 67 +++++ .../snapshot_job_run_value_triggers_test.go | 30 +++ .../mutator/validate_job_run_triggers.go | 9 +- .../mutator/validate_job_run_triggers_test.go | 4 +- bundle/config/resources/job_run.go | 21 +- bundle/direct/dresources/job_run.go | 48 +++- bundle/direct/dresources/job_run_test.go | 9 + bundle/internal/schema/annotations.yml | 3 + bundle/schema/jsonschema.json | 4 + .../job_runs/_models/job_run_trigger.py | 10 + 31 files changed, 855 insertions(+), 19 deletions(-) create mode 100644 .nextchanges/bundles/job-runs-on-value-change.md create mode 100644 acceptance/bundle/resources/job_runs/on_value_change/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_value_change/script create mode 100644 acceptance/bundle/resources/job_runs/on_value_change/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_errors/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_errors/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_errors/script create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_errors/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_errors/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_ref/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_ref/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_ref/script create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml create mode 100644 bundle/config/mutator/snapshot_job_run_value_triggers.go create mode 100644 bundle/config/mutator/snapshot_job_run_value_triggers_test.go diff --git a/.nextchanges/bundles/job-runs-on-value-change.md b/.nextchanges/bundles/job-runs-on-value-change.md new file mode 100644 index 00000000000..dc031aabce5 --- /dev/null +++ b/.nextchanges/bundles/job-runs-on-value-change.md @@ -0,0 +1 @@ +* Add `lifecycle.triggers.on_value_change` for `job_runs`: re-fire the run when a watched expression's resolved value changes. ([#6341](https://github.com/databricks/cli/pull/6341)) diff --git a/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml b/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml new file mode 100644 index 00000000000..cb2511f5c00 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml @@ -0,0 +1,33 @@ +bundle: + name: job-runs-on-value-change + +variables: + stamp: + default: v1 + other: + default: a1 + number: + default: 42 + long: + default: a-long-non-hex-watched-value-used-to-exercise-a-lengthy-string-zzz + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt + - on_value_change: " ${var.stamp} " + - on_value_change: ${var.other} + - on_value_change: ${var.number} + - on_value_change: ${workspace.root_path} + - on_value_change: ${var.long} diff --git a/acceptance/bundle/resources/job_runs/on_value_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_value_change/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/on_value_change/output.txt b/acceptance/bundle/resources/job_runs/on_value_change/output.txt new file mode 100644 index 00000000000..704b6ab71f0 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change/output.txt @@ -0,0 +1,239 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Created jobs.my_job +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== resolved values are stored as hashes +>>> print_state.py +{ + "on_file_change": { + "watched.txt": "[FILE_HASH][0]" + }, + "on_value_change": { + "${var.long}": "a-long-non-hex-watched-value-used-to-exercise-a-lengthy-string-zzz", + "${var.number}": "42", + "${var.other}": "a1", + "${var.stamp}": "v1", + "${workspace.root_path}": "/Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default" + } +} + +=== redeploy with unchanged values plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== changing one value re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle.triggers_state.on_value_change['${var.stamp}']": { + "action": "recreate", + "reason": "immutable", + "old": "v1", + "new": "v2" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== removing one on_value_change through a saved plan does not run +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy --plan tmp.plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== re-adding the same on_value_change after the saved plan re-fires +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> changes +{ + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py --keep //jobs/runs/delete + +>>> print_requests.py //jobs/run-now + +=== removing the same on_value_change again does not run +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== changing a remaining watch to a removed value re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle.triggers_state.on_value_change['${var.stamp}']": { + "action": "recreate", + "reason": "immutable", + "old": "v2", + "new": "a1" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS +Recreated job_runs.my_run +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== removing the remaining on_value_change entries is unchanged and does not run +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== file changes still re-fire after value triggers are removed +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle.triggers_state.on_file_change['watched.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_5]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_5]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_5]: SUCCESS +Recreated job_runs.my_run +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/job_runs/on_value_change/script b/acceptance/bundle/resources/job_runs/on_value_change/script new file mode 100644 index 00000000000..d95ef86a558 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change/script @@ -0,0 +1,87 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "resolved values are stored as hashes" +trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle.triggers_state' + +title "redeploy with unchanged values plans nothing" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "changing one value re-fires" +update_file.py databricks.yml "default: v1" "default: v2" +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "removing one on_value_change through a saved plan does not run" +update_file.py databricks.yml ' - on_value_change: ${var.other} +' '' +trace $CLI bundle plan +$CLI bundle plan -o json > tmp.plan.json +trace $CLI bundle deploy --plan tmp.plan.json +trace print_requests.py //jobs/run-now + +title "re-adding the same on_value_change after the saved plan re-fires" +update_file.py databricks.yml ' - on_value_change: " ${var.stamp} " +' ' - on_value_change: " ${var.stamp} " + - on_value_change: ${var.other} +' +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "removing the same on_value_change again does not run" +update_file.py databricks.yml ' - on_value_change: ${var.other} +' '' +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "changing a remaining watch to a removed value re-fires" +update_file.py databricks.yml "default: v2" "default: a1" +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "removing the remaining on_value_change entries is unchanged and does not run" +update_file.py databricks.yml ' - on_value_change: " ${var.stamp} " + - on_value_change: ${var.number} + - on_value_change: ${workspace.root_path} + - on_value_change: ${var.long} +' '' +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "file changes still re-fire after value triggers are removed" +update_file.py watched.txt "v1" "v2" +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_value_change/test.toml b/acceptance/bundle/resources/job_runs/on_value_change/test.toml new file mode 100644 index 00000000000..608b2e13890 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change/test.toml @@ -0,0 +1,9 @@ +# The on_file_change content hash contains digit runs that the parent `\d{8,}` +# rule would mangle. Order=1 runs before it; Distinct keeps different hashes distinct. +Ignore = ["tmp.plan.json"] + +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_value_change/watched.txt b/acceptance/bundle/resources/job_runs/on_value_change/watched.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change/watched.txt @@ -0,0 +1 @@ +v1 diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_value_change_errors/databricks.yml new file mode 100644 index 00000000000..afd7e8eb28a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: job-runs-on-value-change-errors + +variables: + value: + default: resolved + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: [{on_value_change: " "}] diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/out.test.toml b/acceptance/bundle/resources/job_runs/on_value_change_errors/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt new file mode 100644 index 00000000000..a0ce0866b8d --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt @@ -0,0 +1,51 @@ + +=== an empty expression is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_value_change must be non-empty when set + in databricks.yml:21:38 + +Name: job-runs-on-value-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-errors/default + +Found 1 error + +=== duplicate expressions are rejected after trimming +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_value_change expressions must be unique + in databricks.yml:21:73 + +Name: job-runs-on-value-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-errors/default + +Found 1 error + +=== multiple trigger fields in one entry are rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers entry must set only one of on_bundle_deploy, on_file_change, or on_value_change + in databricks.yml:21:20 + +Name: job-runs-on-value-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-errors/default + +Found 1 error + +=== an undefined variable is rejected +>>> musterr [CLI] bundle validate +Error: reference does not exist: ${var.missing} + +Name: job-runs-on-value-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-errors/default + +Found 1 error diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/script b/acceptance/bundle/resources/job_runs/on_value_change_errors/script new file mode 100644 index 00000000000..ee42ac1ce00 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/script @@ -0,0 +1,14 @@ +title "an empty expression is rejected" +trace musterr $CLI bundle validate + +title "duplicate expressions are rejected after trimming" +update_file.py databricks.yml ' triggers: [{on_value_change: " "}]' ' triggers: [{on_value_change: "${var.value}"}, {on_value_change: " ${var.value} "}]' +trace musterr $CLI bundle validate + +title "multiple trigger fields in one entry are rejected" +update_file.py databricks.yml ' triggers: [{on_value_change: "${var.value}"}, {on_value_change: " ${var.value} "}]' ' triggers: [{on_file_change: watched.txt, on_value_change: "${var.value}"}]' +trace musterr $CLI bundle validate + +title "an undefined variable is rejected" +update_file.py databricks.yml ' triggers: [{on_file_change: watched.txt, on_value_change: "${var.value}"}]' ' triggers: [{on_value_change: "${var.missing}"}]' +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_value_change_errors/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/test.toml @@ -0,0 +1 @@ +RecordRequests = false diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/watched.txt b/acceptance/bundle/resources/job_runs/on_value_change_errors/watched.txt new file mode 100644 index 00000000000..0ed2fb018aa --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/watched.txt @@ -0,0 +1 @@ +watched diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/databricks.yml b/acceptance/bundle/resources/job_runs/on_value_change_ref/databricks.yml new file mode 100644 index 00000000000..f821d3e075c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/databricks.yml @@ -0,0 +1,37 @@ +bundle: + name: job-runs-on-value-change-ref + +variables: + suffix: + default: suffix + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + other: + name: other-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + extra: + name: extra-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_value_change: ${resources.jobs.other.id} + - on_value_change: ${resources.jobs.other.id}-${var.suffix}-${resources.jobs.extra.id} diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/out.test.toml b/acceptance/bundle/resources/job_runs/on_value_change_ref/out.test.toml new file mode 100644 index 00000000000..f7fafbcc731 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt new file mode 100644 index 00000000000..81f5b036137 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt @@ -0,0 +1,107 @@ + +=== plan depends on both jobs in the expressions +>>> jq .plan["resources.job_runs.my_run"].depends_on tmp.plan.json +[ + { + "node": "resources.jobs.extra", + "label": "${resources.jobs.extra.id}" + }, + { + "node": "resources.jobs.my_job", + "label": "${resources.jobs.my_job.id}" + }, + { + "node": "resources.jobs.other", + "label": "${resources.jobs.other.id}" + } +] + +=== first deploy triggers a run +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default/files... +Created jobs.extra +Created jobs.my_job +Created jobs.other +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Files: 2 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> read_id.py other +[OTHER_ID] + +>>> read_id.py extra +[EXTRA_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== state records hashes for pure and mixed references +>>> print_state.py +{ + "${resources.jobs.other.id}": "[OTHER_ID]", + "${resources.jobs.other.id}-${var.suffix}-${resources.jobs.extra.id}": "[OTHER_ID]-suffix-[EXTRA_ID]" +} + +=== redeploy with unchanged ids plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 4 unchanged + +>>> print_requests.py //jobs/run-now + +=== replacing a referenced job re-fires the run +>>> [CLI] jobs delete [OTHER_ID] + +>>> [CLI] bundle plan +recreate job_runs.my_run +create jobs.other + +Plan: 2 to add, 0 to change, 1 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default/files... +Created jobs.other +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 1 deleted, 2 unchanged + +>>> read_id.py other +[OTHER_ID_3] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.extra + delete resources.jobs.my_job + delete resources.jobs.other + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/script b/acceptance/bundle/resources/job_runs/on_value_change_ref/script new file mode 100644 index 00000000000..5eaedb8d6ec --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/script @@ -0,0 +1,38 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "plan depends on both jobs in the expressions" +$CLI bundle plan -o json > tmp.plan.json +trace jq '.plan["resources.job_runs.my_run"].depends_on' tmp.plan.json | contains.py \ + 'resources.jobs.other' '${resources.jobs.other.id}' \ + 'resources.jobs.extra' '${resources.jobs.extra.id}' + +title "first deploy triggers a run" +# Not traced: readplanarg makes the command line differ per READPLAN variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +trace read_id.py my_job +trace read_id.py other +trace read_id.py extra +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "state records hashes for pure and mixed references" +trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle.triggers_state.on_value_change' + +title "redeploy with unchanged ids plans nothing" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "replacing a referenced job re-fires the run" +other_id=$(read_id.py other) +trace $CLI jobs delete "$other_id" +trace $CLI bundle plan +trace $CLI bundle deploy +trace read_id.py other +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml b/acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml new file mode 100644 index 00000000000..803d3f4a8b9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml @@ -0,0 +1,11 @@ +# Deploy both by re-planning and from a plan saved on disk, so resource refs +# in on_value_change survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json"] + +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[VALUE_HASH]' +Order = 1 +Distinct = true diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 35cffd0dcbe..d7db5499bc1 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -72,7 +72,10 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d out[pattern] = fingerprint } } - jr.Lifecycle.TriggersState = &resources.JobRunTriggersState{OnFileChange: out} + if jr.Lifecycle.TriggersState == nil { + jr.Lifecycle.TriggersState = &resources.JobRunTriggersState{} + } + jr.Lifecycle.TriggersState.OnFileChange = out } return diags } diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index 06ec593f092..f6e879f384a 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -41,6 +41,9 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia bundle.ApplySeqContext( ctx, b, + // Preserve on_value_change expressions as stable state keys before interpolation. + mutator.SnapshotJobRunValueTriggers(), + // Reads (dynamic): * (strings) (searches for variable references in string values) // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index e00444cc773..7ec591a2b24 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -139,6 +139,9 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { validate.SingleNodeCluster(), + // Preserve on_value_change expressions as stable state keys before interpolation. + mutator.SnapshotJobRunValueTriggers(), + // Reads (dynamic): * (strings) (searches for variable references in string values) // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes diff --git a/bundle/config/mutator/snapshot_job_run_value_triggers.go b/bundle/config/mutator/snapshot_job_run_value_triggers.go new file mode 100644 index 00000000000..1fb41ad4ef3 --- /dev/null +++ b/bundle/config/mutator/snapshot_job_run_value_triggers.go @@ -0,0 +1,67 @@ +package mutator + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" +) + +type snapshotJobRunValueTriggers struct{} + +// SnapshotJobRunValueTriggers records on_value_change expressions before interpolation. +func SnapshotJobRunValueTriggers() bundle.Mutator { + return &snapshotJobRunValueTriggers{} +} + +func (*snapshotJobRunValueTriggers) Name() string { + return "SnapshotJobRunValueTriggers" +} + +func (*snapshotJobRunValueTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + for name, jr := range b.Config.Resources.JobRuns { + if jr == nil || jr.Lifecycle == nil || + (jr.Lifecycle.TriggersState != nil && jr.Lifecycle.TriggersState.OnValueChange != nil) { + continue + } + + out := make(map[string]string) + for i, trigger := range jr.Lifecycle.Triggers { + if trigger.OnValueChange == nil { + continue + } + + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_value_change", name, i) + expr := strings.TrimSpace(*trigger.OnValueChange) + if expr == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_value_change must be non-empty when set", + Locations: b.Config.GetLocations(path), + }) + continue + } + if _, ok := out[expr]; ok { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_value_change expressions must be unique", + Locations: b.Config.GetLocations(path), + }) + continue + } + out[expr] = expr + } + + if len(out) > 0 { + if jr.Lifecycle.TriggersState == nil { + jr.Lifecycle.TriggersState = &resources.JobRunTriggersState{} + } + jr.Lifecycle.TriggersState.OnValueChange = out + } + } + return diags +} diff --git a/bundle/config/mutator/snapshot_job_run_value_triggers_test.go b/bundle/config/mutator/snapshot_job_run_value_triggers_test.go new file mode 100644 index 00000000000..7d7a7cd5bf5 --- /dev/null +++ b/bundle/config/mutator/snapshot_job_run_value_triggers_test.go @@ -0,0 +1,30 @@ +package mutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" +) + +func TestSnapshotJobRunValueTriggersIsStableAfterInterpolation(t *testing.T) { + expr := "${var.value}" + resolved := "resolved" + jobRun := &resources.JobRun{ + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{{OnValueChange: &expr}}, + }, + } + b := &bundle.Bundle{Config: config.Root{ + Resources: config.Resources{JobRuns: map[string]*resources.JobRun{"run": jobRun}}, + }} + + assert.Empty(t, bundle.Apply(t.Context(), b, mutator.SnapshotJobRunValueTriggers())) + b.Config.Resources.JobRuns["run"].Lifecycle.Triggers[0].OnValueChange = &resolved + assert.Empty(t, bundle.Apply(t.Context(), b, mutator.SnapshotJobRunValueTriggers())) + + assert.Equal(t, map[string]string{expr: expr}, b.Config.Resources.JobRuns["run"].Lifecycle.TriggersState.OnValueChange) +} diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index d33a7263bde..cb12587a306 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -28,18 +28,19 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) - if t.OnBundleDeploy == nil && t.OnFileChange == nil { + armed := t.ArmedCount() + if armed == 0 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + Summary: "lifecycle.triggers entry must set on_bundle_deploy, on_file_change, or on_value_change", Locations: b.Config.GetLocations(path), }) continue } - if t.OnBundleDeploy != nil && t.OnFileChange != nil { + if armed > 1 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", + Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy, on_file_change, or on_value_change", Locations: b.Config.GetLocations(path), }) continue diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 3ec1e90b844..1ab7e9dda5f 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -48,14 +48,14 @@ func TestValidateJobRunTriggers(t *testing.T) { triggers: []resources.JobRunTrigger{ {}, }, - summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + summary: "lifecycle.triggers entry must set on_bundle_deploy, on_file_change, or on_value_change", }, { name: "both keys on one entry", triggers: []resources.JobRunTrigger{ {OnBundleDeploy: &trueVal, OnFileChange: &fileChange}, }, - summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", + summary: "lifecycle.triggers entry must set only one of on_bundle_deploy, on_file_change, or on_value_change", }, { name: "on_bundle_deploy false", diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index dff2e538cb0..5b681f7fe6b 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -44,20 +44,37 @@ type JobRunLifecycle struct { // JobRunTrigger is one lifecycle.triggers entry. type JobRunTrigger struct { OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` - OnFileChange *string `json:"on_file_change,omitempty"` // path or glob relative to the defining YAML file; must resolve under the sync root + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob relative to the defining YAML file; must resolve under the sync root + OnValueChange *string `json:"on_value_change,omitempty"` // expression whose resolved value re-fires the run when changed +} + +// ArmedCount returns how many trigger fields this entry sets. +func (t JobRunTrigger) ArmedCount() int { + n := 0 + if t.OnBundleDeploy != nil { + n++ + } + if t.OnFileChange != nil { + n++ + } + if t.OnValueChange != nil { + n++ + } + return n } // JobRunTriggersState is the resolved fingerprint of lifecycle.triggers. type JobRunTriggersState struct { OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` OnFileChange map[string]string `json:"on_file_change,omitempty"` + OnValueChange map[string]string `json:"on_value_change,omitempty"` } // IsEmpty reports whether no trigger is armed. An empty state is left off the // job run entirely, so this has to cover every field above: a new fingerprint // added without extending it would be dropped instead of persisted. func (s JobRunTriggersState) IsEmpty() bool { - return s.OnBundleDeploy == "" && len(s.OnFileChange) == 0 + return s.OnBundleDeploy == "" && len(s.OnFileChange) == 0 && len(s.OnValueChange) == 0 } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 90a628b0486..51d0af16e88 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "maps" "slices" "strconv" "strings" @@ -14,8 +15,10 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -26,6 +29,10 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour +// on_value_change fingerprints are keyed by the watched expression; the planner +// resolves references at this path, so it is kept as a parsed node. +var jobRunOnValueChangePath = structpath.MustParsePath("lifecycle.triggers_state.on_value_change") + // JobRunLifecycleState is the local-only trigger fingerprint. type JobRunLifecycleState struct { TriggersState *resources.JobRunTriggersState `json:"triggers_state,omitempty"` @@ -96,6 +103,8 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { var ts resources.JobRunTriggersState if input.Lifecycle != nil && input.Lifecycle.TriggersState != nil { ts = *input.Lifecycle.TriggersState + // Fingerprinting below rewrites value-trigger entries. + ts.OnValueChange = maps.Clone(ts.OnValueChange) } if input.HasOnBundleDeploy() { ts.OnBundleDeploy = uuid.New().String() @@ -106,6 +115,22 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { return state } +// PrepareInputConfig maps unresolved watches to state paths so the deploy graph +// tracks their resource references. +func (*ResourceJobRun) PrepareInputConfig(input *resources.JobRun, _ string) (*structvar.StructVar, error) { + refs := map[string]string{} + if input.Lifecycle == nil || input.Lifecycle.TriggersState == nil { + return &structvar.StructVar{Value: input, Refs: refs}, nil + } + for expr, value := range input.Lifecycle.TriggersState.OnValueChange { + if !dynvar.ContainsVariableReference(value) { + continue + } + refs[structpath.NewBracketString(jobRunOnValueChangePath, expr).String()] = value + } + return &structvar.StructVar{Value: input, Refs: refs}, nil +} + // makeJobRunRemote maps the GetRun response into the RunNow-shaped remote: GET // nests the params under overriding_parameters and returns job_parameters as a // list, so both are flattened back into RunNow. @@ -382,10 +407,10 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa if change.New == nil || change.New == "" { change.Reason = deployplan.ReasonDrop } - case "lifecycle.triggers_state.on_file_change": - // As above: an emptied map is a removal, and pattern entries classify a map - // that still has both sides. - if isEmptyFileTriggerMap(change.New) || change.Old != nil { + case "lifecycle.triggers_state.on_file_change", "lifecycle.triggers_state.on_value_change": + // An emptied map is a removal; a map with both sides present has its + // per-key entries (below) classify the change. + if isEmptyTriggerMap(change.New) || change.Old != nil { change.Reason = deployplan.ReasonDrop } case "result_state": @@ -396,12 +421,15 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Action = deployplan.Skip change.Reason = "run in progress" default: - // A single on_file_change pattern entry, e.g. - // lifecycle.triggers_state.on_file_change['seed.txt']. Removing one pattern + // A single on_file_change pattern or on_value_change expression entry, e.g. + // lifecycle.triggers_state.on_file_change['seed.txt']. Removing one entry // drops its change and leaves the last fingerprint in state. - if parent := path.Parent(); parent != nil && parent.String() == "lifecycle.triggers_state.on_file_change" { - if change.New == nil { - change.Reason = deployplan.ReasonDrop + if parent := path.Parent(); parent != nil { + switch parent.String() { + case "lifecycle.triggers_state.on_file_change", "lifecycle.triggers_state.on_value_change": + if change.New == nil { + change.Reason = deployplan.ReasonDrop + } } } } @@ -413,7 +441,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa // re-fires) leaves the run alone on the backend and only drops the state entry // (see PlanEntry.StateOnly). A recreate still triggers a fresh run. -func isEmptyFileTriggerMap(v any) bool { +func isEmptyTriggerMap(v any) bool { if v == nil { return true } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 5dc964c360f..4d3a25dbf2a 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -272,6 +272,9 @@ func TestJobRunPrepareStateCopiesResolvedTriggers(t *testing.T) { enabled := true triggers := &resources.JobRunTriggersState{ OnFileChange: map[string]string{"*.txt": "hash"}, + OnValueChange: map[string]string{ + "${var.watched}": "resolved", + }, } input := &resources.JobRun{ Lifecycle: &resources.JobRunLifecycle{ @@ -286,6 +289,12 @@ func TestJobRunPrepareStateCopiesResolvedTriggers(t *testing.T) { require.NotNil(t, state.Lifecycle.TriggersState) assert.NotSame(t, triggers, state.Lifecycle.TriggersState) assert.Equal(t, triggers.OnFileChange, state.Lifecycle.TriggersState.OnFileChange) + assert.Equal(t, map[string]string{ + "${var.watched}": "resolved", + }, state.Lifecycle.TriggersState.OnValueChange) + assert.Equal(t, map[string]string{ + "${var.watched}": "resolved", + }, triggers.OnValueChange) assert.NotEmpty(t, state.Lifecycle.TriggersState.OnBundleDeploy) assert.Empty(t, triggers.OnBundleDeploy) } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 08b65053e3b..d991e155ec3 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1133,6 +1133,9 @@ resources: "on_file_change": "description": |- Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. + "on_value_change": + "description": |- + Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Resolved values are recorded as SHA-256 hashes. Removing a watch disarms it without re-firing the run. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index a060ec2e8a1..2d313084e6a 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1708,6 +1708,10 @@ "on_file_change": { "description": "Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported.", "$ref": "#/$defs/string" + }, + "on_value_change": { + "description": "Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Resolved values are recorded as SHA-256 hashes. Removing a watch disarms it without re-firing the run. Incompatible with lifecycle.prevent_destroy.", + "$ref": "#/$defs/string" } }, "additionalProperties": false diff --git a/python/databricks/bundles/job_runs/_models/job_run_trigger.py b/python/databricks/bundles/job_runs/_models/job_run_trigger.py index 75bdf970a4e..2561e8d8b0d 100644 --- a/python/databricks/bundles/job_runs/_models/job_run_trigger.py +++ b/python/databricks/bundles/job_runs/_models/job_run_trigger.py @@ -25,6 +25,11 @@ class JobRunTrigger: Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. """ + on_value_change: VariableOrOptional[str] = None + """ + Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Resolved values are recorded as SHA-256 hashes. Removing a watch disarms it without re-firing the run. Incompatible with lifecycle.prevent_destroy. + """ + @classmethod def from_dict(cls, value: "JobRunTriggerDict") -> "Self": return _transform(cls, value) @@ -46,5 +51,10 @@ class JobRunTriggerDict(TypedDict, total=False): Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. """ + on_value_change: VariableOrOptional[str] + """ + Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Resolved values are recorded as SHA-256 hashes. Removing a watch disarms it without re-firing the run. Incompatible with lifecycle.prevent_destroy. + """ + JobRunTriggerParam = JobRunTriggerDict | JobRunTrigger From ecae894c8b763d88cfc9878e91479db96faf80cb Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 11:23:02 +0100 Subject: [PATCH 02/12] Fix stale on_value_change docs: values are stored raw, prevent_destroy is allowed Co-authored-by: Isaac --- .../bundle/resources/job_runs/on_value_change/output.txt | 2 +- acceptance/bundle/resources/job_runs/on_value_change/script | 2 +- .../bundle/resources/job_runs/on_value_change_ref/output.txt | 2 +- .../bundle/resources/job_runs/on_value_change_ref/script | 2 +- bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- python/databricks/bundles/job_runs/_models/job_run_trigger.py | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_value_change/output.txt b/acceptance/bundle/resources/job_runs/on_value_change/output.txt index 704b6ab71f0..f9f31ad7f5f 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change/output.txt @@ -22,7 +22,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== resolved values are stored as hashes +=== resolved values are stored in state >>> print_state.py { "on_file_change": { diff --git a/acceptance/bundle/resources/job_runs/on_value_change/script b/acceptance/bundle/resources/job_runs/on_value_change/script index d95ef86a558..46952327e26 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/script +++ b/acceptance/bundle/resources/job_runs/on_value_change/script @@ -14,7 +14,7 @@ trace read_id.py my_job read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -title "resolved values are stored as hashes" +title "resolved values are stored in state" trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle.triggers_state' title "redeploy with unchanged values plans nothing" diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt index 81f5b036137..19425af4471 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt @@ -47,7 +47,7 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged } } -=== state records hashes for pure and mixed references +=== state records resolved values for pure and mixed references >>> print_state.py { "${resources.jobs.other.id}": "[OTHER_ID]", diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/script b/acceptance/bundle/resources/job_runs/on_value_change_ref/script index 5eaedb8d6ec..4e04db3fff5 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_ref/script +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/script @@ -20,7 +20,7 @@ trace read_id.py extra read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -title "state records hashes for pure and mixed references" +title "state records resolved values for pure and mixed references" trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle.triggers_state.on_value_change' title "redeploy with unchanged ids plans nothing" diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index d991e155ec3..07023f7148f 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1135,7 +1135,7 @@ resources: Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. "on_value_change": "description": |- - Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Resolved values are recorded as SHA-256 hashes. Removing a watch disarms it without re-firing the run. Incompatible with lifecycle.prevent_destroy. + Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Removing a watch disarms it without re-firing the run. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 2d313084e6a..00c29b156f7 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1710,7 +1710,7 @@ "$ref": "#/$defs/string" }, "on_value_change": { - "description": "Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Resolved values are recorded as SHA-256 hashes. Removing a watch disarms it without re-firing the run. Incompatible with lifecycle.prevent_destroy.", + "description": "Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Removing a watch disarms it without re-firing the run.", "$ref": "#/$defs/string" } }, diff --git a/python/databricks/bundles/job_runs/_models/job_run_trigger.py b/python/databricks/bundles/job_runs/_models/job_run_trigger.py index 2561e8d8b0d..84c12cf7a24 100644 --- a/python/databricks/bundles/job_runs/_models/job_run_trigger.py +++ b/python/databricks/bundles/job_runs/_models/job_run_trigger.py @@ -27,7 +27,7 @@ class JobRunTrigger: on_value_change: VariableOrOptional[str] = None """ - Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Resolved values are recorded as SHA-256 hashes. Removing a watch disarms it without re-firing the run. Incompatible with lifecycle.prevent_destroy. + Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Removing a watch disarms it without re-firing the run. """ @classmethod @@ -53,7 +53,7 @@ class JobRunTriggerDict(TypedDict, total=False): on_value_change: VariableOrOptional[str] """ - Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Resolved values are recorded as SHA-256 hashes. Removing a watch disarms it without re-firing the run. Incompatible with lifecycle.prevent_destroy. + Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Removing a watch disarms it without re-firing the run. """ From 94850c1462f94de2ed68d185f96d0a50061ff6b3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 11:29:30 +0100 Subject: [PATCH 03/12] Update goldens after rebase on main Co-authored-by: Isaac --- acceptance/bundle/refschema/output/job_runs.txt | 3 +++ .../bundle/resources/job_runs/on_value_change/output.txt | 8 ++++---- .../resources/job_runs/on_value_change_ref/output.txt | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/refschema/output/job_runs.txt b/acceptance/bundle/refschema/output/job_runs.txt index c613f750f7b..cfeca5028b9 100644 --- a/acceptance/bundle/refschema/output/job_runs.txt +++ b/acceptance/bundle/refschema/output/job_runs.txt @@ -19,10 +19,13 @@ resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT +resources.job_runs.*.lifecycle.triggers[*].on_value_change *string INPUT resources.job_runs.*.lifecycle.triggers_state *resources.JobRunTriggersState INPUT STATE resources.job_runs.*.lifecycle.triggers_state.on_bundle_deploy string INPUT STATE resources.job_runs.*.lifecycle.triggers_state.on_file_change map[string]string INPUT STATE resources.job_runs.*.lifecycle.triggers_state.on_file_change.* string INPUT STATE +resources.job_runs.*.lifecycle.triggers_state.on_value_change map[string]string INPUT STATE +resources.job_runs.*.lifecycle.triggers_state.on_value_change.* string INPUT STATE resources.job_runs.*.modified_status string INPUT resources.job_runs.*.notebook_params map[string]string ALL resources.job_runs.*.notebook_params.* string ALL diff --git a/acceptance/bundle/resources/job_runs/on_value_change/output.txt b/acceptance/bundle/resources/job_runs/on_value_change/output.txt index f9f31ad7f5f..933acd095a5 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... Created jobs.my_job -Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Files: 2 uploaded, 0 deleted @@ -72,7 +72,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 1 uploaded, 0 deleted @@ -159,7 +159,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS Recreated job_runs.my_run Files: 1 uploaded, 0 deleted @@ -212,7 +212,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_5]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_5]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_5]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_5]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID_5]: SUCCESS Recreated job_runs.my_run Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt index 19425af4471..9b4ccfe9c17 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt @@ -22,7 +22,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value- Created jobs.extra Created jobs.my_job Created jobs.other -Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Files: 2 uploaded, 0 deleted @@ -77,7 +77,7 @@ Plan: 2 to add, 0 to change, 1 to delete, 2 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default/files... Created jobs.other -Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[WORKSPACE_ID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 0 uploaded, 0 deleted From d9de37b216d247b05564a9b02ac5ead3581b8981 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 13:05:12 +0100 Subject: [PATCH 04/12] Add on_value_change tests recording current failures Record today's behavior, including the bugs: - python/job_runs-on_value_change: any bundle with a python: section and an on_value_change watch fails validate with "Unexpected field 'triggers_state' for class JobRunLifecycle", for YAML- and Python-defined job_runs alike. - on_value_change_empty: a watched variable whose value is "" fails with "lifecycle.triggers entry must set ..." on an entry that does set it. - on_value_change_errors: a complex variable gives two "expected string, found map" warnings (one at the internal triggers_state path, at :0:0) and the same misleading error. - on_value_change_ref: updating a referenced job in place does not re-fire the run (correct; locks it in). Co-authored-by: Isaac --- .../job_runs-on_value_change/databricks.yml | 32 ++++++++++++ .../job_runs-on_value_change/mutators.py | 12 +++++ .../job_runs-on_value_change/out.test.toml | 3 ++ .../job_runs-on_value_change/output.txt | 51 +++++++++++++++++++ .../job_runs-on_value_change/resources.py | 15 ++++++ .../python/job_runs-on_value_change/script | 21 ++++++++ .../python/job_runs-on_value_change/test.toml | 12 +++++ .../on_value_change_empty/databricks.yml | 21 ++++++++ .../on_value_change_empty/out.test.toml | 2 + .../job_runs/on_value_change_empty/output.txt | 13 +++++ .../job_runs/on_value_change_empty/script | 28 ++++++++++ .../on_value_change_errors/databricks.yml | 3 ++ .../on_value_change_errors/output.txt | 29 +++++++++-- .../job_runs/on_value_change_errors/script | 4 ++ .../job_runs/on_value_change_ref/output.txt | 14 +++++ .../job_runs/on_value_change_ref/script | 6 +++ 16 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 acceptance/bundle/python/job_runs-on_value_change/databricks.yml create mode 100644 acceptance/bundle/python/job_runs-on_value_change/mutators.py create mode 100644 acceptance/bundle/python/job_runs-on_value_change/out.test.toml create mode 100644 acceptance/bundle/python/job_runs-on_value_change/output.txt create mode 100644 acceptance/bundle/python/job_runs-on_value_change/resources.py create mode 100644 acceptance/bundle/python/job_runs-on_value_change/script create mode 100644 acceptance/bundle/python/job_runs-on_value_change/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_empty/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_empty/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_empty/script diff --git a/acceptance/bundle/python/job_runs-on_value_change/databricks.yml b/acceptance/bundle/python/job_runs-on_value_change/databricks.yml new file mode 100644 index 00000000000..dca8ab672c0 --- /dev/null +++ b/acceptance/bundle/python/job_runs-on_value_change/databricks.yml @@ -0,0 +1,32 @@ +bundle: + name: test-bundle + +sync: {paths: []} # don't need to copy files + +variables: + x: + default: x1 + y: + default: y1 + +python: + resources: + - "resources:load_resources" + mutators: + - "mutators:watch_y" + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + yaml_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_value_change: ${var.x} diff --git a/acceptance/bundle/python/job_runs-on_value_change/mutators.py b/acceptance/bundle/python/job_runs-on_value_change/mutators.py new file mode 100644 index 00000000000..15ef3dc2f58 --- /dev/null +++ b/acceptance/bundle/python/job_runs-on_value_change/mutators.py @@ -0,0 +1,12 @@ +from dataclasses import replace + +from databricks.bundles.core import job_run_mutator +from databricks.bundles.job_runs import JobRun, JobRunLifecycle, JobRunTrigger + + +@job_run_mutator +def watch_y(run: JobRun) -> JobRun: + lifecycle = run.lifecycle or JobRunLifecycle() + triggers = [*lifecycle.triggers, JobRunTrigger(on_value_change="${var.y}")] + + return replace(run, lifecycle=replace(lifecycle, triggers=triggers)) diff --git a/acceptance/bundle/python/job_runs-on_value_change/out.test.toml b/acceptance/bundle/python/job_runs-on_value_change/out.test.toml new file mode 100644 index 00000000000..aeddf3e619e --- /dev/null +++ b/acceptance/bundle/python/job_runs-on_value_change/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/job_runs-on_value_change/output.txt b/acceptance/bundle/python/job_runs-on_value_change/output.txt new file mode 100644 index 00000000000..97ae3fa0a66 --- /dev/null +++ b/acceptance/bundle/python/job_runs-on_value_change/output.txt @@ -0,0 +1,51 @@ + +=== validate keeps a watch from YAML, from Python and from a Python mutator +>>> uv run [UV_ARGS] -q [CLI] bundle validate --output json +Error: Error while loading job_run + at resources.job_runs.py_run + +[TRACEBACK] +ValueError: Unexpected field 'triggers_state' for class JobRunLifecycle + + +Error: Error while loading job_run + at resources.job_runs.yaml_run + +[TRACEBACK] +ValueError: Unexpected field 'triggers_state' for class JobRunLifecycle + + +{ + "py_run": { + "job_id": "${resources.jobs.my_job.id}", + "lifecycle": { + "triggers": [ + { + "on_value_change": "x1" + } + ], + "triggers_state": { + "on_value_change": { + "${var.x}": "x1" + } + } + } + }, + "yaml_run": { + "job_id": "${resources.jobs.my_job.id}", + "lifecycle": { + "triggers": [ + { + "on_value_change": "x1" + } + ], + "triggers_state": { + "on_value_change": { + "${var.x}": "x1" + } + } + } + } +} + +Exit code: 1 diff --git a/acceptance/bundle/python/job_runs-on_value_change/resources.py b/acceptance/bundle/python/job_runs-on_value_change/resources.py new file mode 100644 index 00000000000..73c4f39944f --- /dev/null +++ b/acceptance/bundle/python/job_runs-on_value_change/resources.py @@ -0,0 +1,15 @@ +from databricks.bundles.core import Resources + + +def load_resources() -> Resources: + resources = Resources() + + resources.add_job_run( + "py_run", + { + "job_id": "${resources.jobs.my_job.id}", + "lifecycle": {"triggers": [{"on_value_change": "${var.x}"}]}, + }, + ) + + return resources diff --git a/acceptance/bundle/python/job_runs-on_value_change/script b/acceptance/bundle/python/job_runs-on_value_change/script new file mode 100644 index 00000000000..acf3475010b --- /dev/null +++ b/acceptance/bundle/python/job_runs-on_value_change/script @@ -0,0 +1,21 @@ +title "validate keeps a watch from YAML, from Python and from a Python mutator" +trace uv run $UV_ARGS -q $CLI bundle validate --output json | jq .resources.job_runs + +title "first deploy runs both" +trace uv run $UV_ARGS -q $CLI bundle deploy +trace print_state.py | jq '.state["resources.job_runs.py_run"].state.lifecycle, .state["resources.job_runs.yaml_run"].state.lifecycle' + +title "redeploy with unchanged values plans nothing" +trace uv run $UV_ARGS -q $CLI bundle plan + +title "changing the variable watched in YAML and Python re-fires both runs" +update_file.py databricks.yml "default: x1" "default: x2" +trace uv run $UV_ARGS -q $CLI bundle plan +trace uv run $UV_ARGS -q $CLI bundle deploy + +title "changing the variable the Python mutator watches re-fires both runs" +update_file.py databricks.yml "default: y1" "default: y2" +trace uv run $UV_ARGS -q $CLI bundle plan +trace uv run $UV_ARGS -q $CLI bundle deploy + +rm -fr .databricks __pycache__ diff --git a/acceptance/bundle/python/job_runs-on_value_change/test.toml b/acceptance/bundle/python/job_runs-on_value_change/test.toml new file mode 100644 index 00000000000..f602f61bef1 --- /dev/null +++ b/acceptance/bundle/python/job_runs-on_value_change/test.toml @@ -0,0 +1,12 @@ +Cloud = false # tests don't interact with APIs + +# job_runs are only supported in the current version of the wheel +EnvMatrix.PYDAB_VERSION = ["current"] + +# Both runs fire in parallel, so their progress lines interleave in any order. +SuppressLines.run_output = '^Output from job_runs\.' + +# Python tracebacks name the local site-packages path and vary by Python version. +[[Repls]] +Old = '(?s)Traceback \(most recent call last\):\n.*?\n(ValueError: )' +New = "[TRACEBACK]\n$1" diff --git a/acceptance/bundle/resources/job_runs/on_value_change_empty/databricks.yml b/acceptance/bundle/resources/job_runs/on_value_change_empty/databricks.yml new file mode 100644 index 00000000000..25c1444bfca --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_empty/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: test-bundle + +variables: + optional: {default: ""} + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_value_change: ${var.optional} diff --git a/acceptance/bundle/resources/job_runs/on_value_change_empty/out.test.toml b/acceptance/bundle/resources/job_runs/on_value_change_empty/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_empty/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt new file mode 100644 index 00000000000..8dcebaa5c4d --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt @@ -0,0 +1,13 @@ + +=== a watch on an empty value deploys and records it +>>> [CLI] bundle deploy +Error: lifecycle.triggers entry must set on_bundle_deploy, on_file_change, or on_value_change + in databricks.yml:21:13 + + +>>> [CLI] bundle destroy --auto-approve +Error: lifecycle.triggers entry must set on_bundle_deploy, on_file_change, or on_value_change + in databricks.yml:21:13 + + +Exit code: 1 diff --git a/acceptance/bundle/resources/job_runs/on_value_change_empty/script b/acceptance/bundle/resources/job_runs/on_value_change_empty/script new file mode 100644 index 00000000000..ed637057480 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_empty/script @@ -0,0 +1,28 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "a watch on an empty value deploys and records it" +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now +trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle' + +title "redeploy with the value still empty plans nothing" +trace $CLI bundle plan + +title "setting the value re-fires" +update_file.py databricks.yml '{default: ""}' '{default: set}' +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "clearing the value again re-fires" +update_file.py databricks.yml '{default: set}' '{default: ""}' +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_value_change_errors/databricks.yml index afd7e8eb28a..b8eb272e06a 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_errors/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/databricks.yml @@ -4,6 +4,9 @@ bundle: variables: value: default: resolved + complex: + type: complex + default: {key: value} resources: jobs: diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt index a0ce0866b8d..cdbd542483b 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt @@ -2,7 +2,7 @@ === an empty expression is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_value_change must be non-empty when set - in databricks.yml:21:38 + in databricks.yml:24:38 Name: job-runs-on-value-change-errors Target: default @@ -15,7 +15,7 @@ Found 1 error === duplicate expressions are rejected after trimming >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_value_change expressions must be unique - in databricks.yml:21:73 + in databricks.yml:24:73 Name: job-runs-on-value-change-errors Target: default @@ -28,7 +28,7 @@ Found 1 error === multiple trigger fields in one entry are rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers entry must set only one of on_bundle_deploy, on_file_change, or on_value_change - in databricks.yml:21:20 + in databricks.yml:24:20 Name: job-runs-on-value-change-errors Target: default @@ -49,3 +49,26 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-errors/default Found 1 error + +=== a complex value is rejected +>>> [CLI] bundle validate +Warning: expected string, found map + at resources.job_runs.my_run.lifecycle.triggers[0].on_value_change + in databricks.yml:24:38 + +Warning: expected string, found map + at resources.job_runs.my_run.lifecycle.triggers_state.on_value_change['${var.complex}'] + in :0:0 + +Error: lifecycle.triggers entry must set on_bundle_deploy, on_file_change, or on_value_change + in databricks.yml:24:20 + +Name: job-runs-on-value-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-errors/default + +Found 1 error and 2 warnings + +Exit code: 1 diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/script b/acceptance/bundle/resources/job_runs/on_value_change_errors/script index ee42ac1ce00..f9ca5adde8a 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/script @@ -12,3 +12,7 @@ trace musterr $CLI bundle validate title "an undefined variable is rejected" update_file.py databricks.yml ' triggers: [{on_file_change: watched.txt, on_value_change: "${var.value}"}]' ' triggers: [{on_value_change: "${var.missing}"}]' trace musterr $CLI bundle validate + +title "a complex value is rejected" +update_file.py databricks.yml ' triggers: [{on_value_change: "${var.missing}"}]' ' triggers: [{on_value_change: "${var.complex}"}]' +errcode trace $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt index 9b4ccfe9c17..4dec3f6d6b1 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt @@ -65,6 +65,20 @@ Resources: 0 created, 0 changed, 0 deleted, 4 unchanged >>> print_requests.py //jobs/run-now +=== updating a referenced job in place does not re-fire the run +>>> [CLI] bundle plan +update jobs.other + +Plan: 0 to add, 1 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default/files... +Updated jobs.other +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 3 unchanged + +>>> print_requests.py //jobs/run-now + === replacing a referenced job re-fires the run >>> [CLI] jobs delete [OTHER_ID] diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/script b/acceptance/bundle/resources/job_runs/on_value_change_ref/script index 4e04db3fff5..c3aedf9ef51 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_ref/script +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/script @@ -28,6 +28,12 @@ trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //jobs/run-now +title "updating a referenced job in place does not re-fire the run" +update_file.py databricks.yml "name: other-job" "name: other-job-renamed" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + title "replacing a referenced job re-fires the run" other_id=$(read_id.py other) trace $CLI jobs delete "$other_id" From 53761cfb821436b53bd1c35d7438afdba5ec4d2d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 13:15:09 +0100 Subject: [PATCH 05/12] Snapshot on_value_change after PythonMutator SnapshotJobRunValueTriggers ran in ProcessStaticResources, before PythonMutator, so Python saw the internal lifecycle.triggers_state, which the PyDABs JobRunLifecycle model does not declare: any bundle with a python: section and an on_value_change watch failed with "Unexpected field 'triggers_state' for class JobRunLifecycle". on_value_change expressions are now never resolved in place (dynvar.ResolveExcept, keepUnresolved), so the expression survives until after PythonMutator, where it is copied into triggers_state and resolved there (ResolveJobRunValueTriggerReferences). This also lets a Python mutator add a watch, drops the "already snapshotted" guard, and fixes a watched value of "": the trigger entry keeps its expression, so DropEmptyStrings no longer empties it. A complex value now only warns; the next commit rejects it. Co-authored-by: Isaac --- .../job_runs-on_value_change/output.txt | 71 +++++++++++---- .../python/job_runs-on_value_change/script | 1 - .../python/job_runs-on_value_change/test.toml | 5 -- .../job_runs/on_value_change_empty/output.txt | 86 +++++++++++++++++-- .../job_runs/on_value_change_empty/script | 1 + .../on_value_change_errors/output.txt | 11 +-- .../mutator/resolve_variable_references.go | 36 +++++++- .../process_static_resources.go | 3 - .../resourcemutator/resource_mutator.go | 3 - .../snapshot_job_run_value_triggers.go | 9 +- bundle/phases/initialize.go | 8 ++ libs/dyn/dynvar/resolve.go | 14 ++- libs/dyn/dynvar/resolve_test.go | 16 ++++ 13 files changed, 211 insertions(+), 53 deletions(-) diff --git a/acceptance/bundle/python/job_runs-on_value_change/output.txt b/acceptance/bundle/python/job_runs-on_value_change/output.txt index 97ae3fa0a66..ee296d0028f 100644 --- a/acceptance/bundle/python/job_runs-on_value_change/output.txt +++ b/acceptance/bundle/python/job_runs-on_value_change/output.txt @@ -1,32 +1,22 @@ === validate keeps a watch from YAML, from Python and from a Python mutator >>> uv run [UV_ARGS] -q [CLI] bundle validate --output json -Error: Error while loading job_run - at resources.job_runs.py_run - -[TRACEBACK] -ValueError: Unexpected field 'triggers_state' for class JobRunLifecycle - - -Error: Error while loading job_run - at resources.job_runs.yaml_run - -[TRACEBACK] -ValueError: Unexpected field 'triggers_state' for class JobRunLifecycle - - { "py_run": { "job_id": "${resources.jobs.my_job.id}", "lifecycle": { "triggers": [ { - "on_value_change": "x1" + "on_value_change": "${var.x}" + }, + { + "on_value_change": "${var.y}" } ], "triggers_state": { "on_value_change": { - "${var.x}": "x1" + "${var.x}": "x1", + "${var.y}": "y1" } } } @@ -36,16 +26,59 @@ ValueError: Unexpected field 'triggers_state' for class JobRunLifecycle "lifecycle": { "triggers": [ { - "on_value_change": "x1" + "on_value_change": "${var.x}" + }, + { + "on_value_change": "${var.y}" } ], "triggers_state": { "on_value_change": { - "${var.x}": "x1" + "${var.x}": "x1", + "${var.y}": "y1" } } } } } -Exit code: 1 +=== first deploy runs both +>>> uv run [UV_ARGS] -q [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created job_runs.py_run +Created job_runs.yaml_run +Created jobs.my_job +Files: 0 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== redeploy with unchanged values plans nothing +>>> uv run [UV_ARGS] -q [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +=== changing the variable watched in YAML and Python re-fires both runs +>>> uv run [UV_ARGS] -q [CLI] bundle plan +recreate job_runs.py_run +recreate job_runs.yaml_run + +Plan: 2 to add, 0 to change, 2 to delete, 1 unchanged + +>>> uv run [UV_ARGS] -q [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Recreated job_runs.py_run +Recreated job_runs.yaml_run +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 2 deleted, 1 unchanged + +=== changing the variable the Python mutator watches re-fires both runs +>>> uv run [UV_ARGS] -q [CLI] bundle plan +recreate job_runs.py_run +recreate job_runs.yaml_run + +Plan: 2 to add, 0 to change, 2 to delete, 1 unchanged + +>>> uv run [UV_ARGS] -q [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Recreated job_runs.py_run +Recreated job_runs.yaml_run +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 2 deleted, 1 unchanged diff --git a/acceptance/bundle/python/job_runs-on_value_change/script b/acceptance/bundle/python/job_runs-on_value_change/script index acf3475010b..94bad42f2c3 100644 --- a/acceptance/bundle/python/job_runs-on_value_change/script +++ b/acceptance/bundle/python/job_runs-on_value_change/script @@ -3,7 +3,6 @@ trace uv run $UV_ARGS -q $CLI bundle validate --output json | jq .resources.job_ title "first deploy runs both" trace uv run $UV_ARGS -q $CLI bundle deploy -trace print_state.py | jq '.state["resources.job_runs.py_run"].state.lifecycle, .state["resources.job_runs.yaml_run"].state.lifecycle' title "redeploy with unchanged values plans nothing" trace uv run $UV_ARGS -q $CLI bundle plan diff --git a/acceptance/bundle/python/job_runs-on_value_change/test.toml b/acceptance/bundle/python/job_runs-on_value_change/test.toml index f602f61bef1..f99876d6945 100644 --- a/acceptance/bundle/python/job_runs-on_value_change/test.toml +++ b/acceptance/bundle/python/job_runs-on_value_change/test.toml @@ -5,8 +5,3 @@ EnvMatrix.PYDAB_VERSION = ["current"] # Both runs fire in parallel, so their progress lines interleave in any order. SuppressLines.run_output = '^Output from job_runs\.' - -# Python tracebacks name the local site-packages path and vary by Python version. -[[Repls]] -Old = '(?s)Traceback \(most recent call last\):\n.*?\n(ValueError: )' -New = "[TRACEBACK]\n$1" diff --git a/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt index 8dcebaa5c4d..8e2ef7cbae1 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt @@ -1,13 +1,89 @@ === a watch on an empty value deploys and records it >>> [CLI] bundle deploy -Error: lifecycle.triggers entry must set on_bundle_deploy, on_file_change, or on_value_change - in databricks.yml:21:13 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.my_job +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[WORKSPACE_ID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +>>> print_state.py +{ + "triggers_state": { + "on_value_change": { + "${var.optional}": "" + } + } +} + +=== redeploy with the value still empty plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== setting the value re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[WORKSPACE_ID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== clearing the value again re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[WORKSPACE_ID] +Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS +Recreated job_runs.my_run +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} >>> [CLI] bundle destroy --auto-approve -Error: lifecycle.triggers entry must set on_bundle_deploy, on_file_change, or on_value_change - in databricks.yml:21:13 +The following resources will be deleted: + delete resources.jobs.my_job +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default -Exit code: 1 +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/job_runs/on_value_change_empty/script b/acceptance/bundle/resources/job_runs/on_value_change_empty/script index ed637057480..d0e9aca93bd 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_empty/script +++ b/acceptance/bundle/resources/job_runs/on_value_change_empty/script @@ -6,6 +6,7 @@ trap cleanup EXIT title "a watch on an empty value deploys and records it" trace $CLI bundle deploy +read_id.py my_job > /dev/null read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle' diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt index cdbd542483b..4bac8f29791 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt @@ -52,23 +52,14 @@ Found 1 error === a complex value is rejected >>> [CLI] bundle validate -Warning: expected string, found map - at resources.job_runs.my_run.lifecycle.triggers[0].on_value_change - in databricks.yml:24:38 - Warning: expected string, found map at resources.job_runs.my_run.lifecycle.triggers_state.on_value_change['${var.complex}'] in :0:0 -Error: lifecycle.triggers entry must set on_bundle_deploy, on_file_change, or on_value_change - in databricks.yml:24:20 - Name: job-runs-on-value-change-errors Target: default Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-errors/default -Found 1 error and 2 warnings - -Exit code: 1 +Found 1 warning diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index a5e0d3bf086..1a85396d064 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -48,6 +48,21 @@ var defaultPrefixes = []string{ var artifactPath = dyn.MustPathFromString("artifacts") +// keepUnresolved holds the locations whose references are never resolved in place. +// +// A job_runs on_value_change expression is the key its resolved value is recorded +// under, so it has to survive until SnapshotJobRunValueTriggers copies it into +// triggers_state after PythonMutator; ResolveJobRunValueTriggerReferences then +// resolves the copy. +var keepUnresolved = func() *dyn.TrieNode { + trie := &dyn.TrieNode{} + err := trie.Insert(dyn.MustPatternFromString("resources.job_runs.*.lifecycle.triggers[*].on_value_change")) + if err != nil { + panic(err) + } + return trie +}() + type resolveVariableReferences struct { prefixes []string pattern dyn.Pattern @@ -103,6 +118,18 @@ func ResolveVariableReferencesInLookup() bundle.Mutator { } } +// ResolveJobRunValueTriggerReferences resolves the on_value_change expressions that +// SnapshotJobRunValueTriggers copied into resources.job_runs.*.lifecycle.triggers_state. +func ResolveJobRunValueTriggerReferences() bundle.Mutator { + return &resolveVariableReferences{ + prefixes: defaultPrefixes, + lookupFn: lookup, + extraRounds: maxResolutionRounds - 1, + pattern: dyn.MustPatternFromString("resources.job_runs.*.lifecycle.triggers_state.on_value_change"), + includeResources: true, + } +} + // ResolveVolumePathReferencesOnlyResources resolves only references to resources.volumes.*.volume_path. func ResolveVolumePathReferencesOnlyResources() bundle.Mutator { return &resolveVariableReferences{ @@ -261,8 +288,13 @@ func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn // If the pattern is nil, we resolve references in the entire configuration. root, err := dyn.MapByPattern(root, m.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + kept := func(rel dyn.Path) bool { + _, ok := keepUnresolved.SearchPath(slices.Concat(p, rel)) + return ok + } + // Resolve variable references in all values. - return dynvar.Resolve(v, func(path dyn.Path) (dyn.Value, error) { + return dynvar.ResolveExcept(v, func(path dyn.Path) (dyn.Value, error) { // Rewrite the shorthand path ${var.foo} into ${variables.foo.value}. if path.HasPrefix(varPath) { newPath := dyn.NewPath( @@ -297,7 +329,7 @@ func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn } return dyn.InvalidValue, dynvar.ErrSkipResolution - }) + }, kept) }) if err != nil { return dyn.InvalidValue, err diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index f6e879f384a..06ec593f092 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -41,9 +41,6 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia bundle.ApplySeqContext( ctx, b, - // Preserve on_value_change expressions as stable state keys before interpolation. - mutator.SnapshotJobRunValueTriggers(), - // Reads (dynamic): * (strings) (searches for variable references in string values) // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 7ec591a2b24..e00444cc773 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -139,9 +139,6 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { validate.SingleNodeCluster(), - // Preserve on_value_change expressions as stable state keys before interpolation. - mutator.SnapshotJobRunValueTriggers(), - // Reads (dynamic): * (strings) (searches for variable references in string values) // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes diff --git a/bundle/config/mutator/snapshot_job_run_value_triggers.go b/bundle/config/mutator/snapshot_job_run_value_triggers.go index 1fb41ad4ef3..015aae40f41 100644 --- a/bundle/config/mutator/snapshot_job_run_value_triggers.go +++ b/bundle/config/mutator/snapshot_job_run_value_triggers.go @@ -12,7 +12,11 @@ import ( type snapshotJobRunValueTriggers struct{} -// SnapshotJobRunValueTriggers records on_value_change expressions before interpolation. +// SnapshotJobRunValueTriggers copies each on_value_change expression into +// lifecycle.triggers_state, keyed by the trimmed expression, for +// ResolveJobRunValueTriggerReferences to resolve. The expression itself is never +// resolved in place (see keepUnresolved), so the key stays stable when the value +// changes. func SnapshotJobRunValueTriggers() bundle.Mutator { return &snapshotJobRunValueTriggers{} } @@ -24,8 +28,7 @@ func (*snapshotJobRunValueTriggers) Name() string { func (*snapshotJobRunValueTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics for name, jr := range b.Config.Resources.JobRuns { - if jr == nil || jr.Lifecycle == nil || - (jr.Lifecycle.TriggersState != nil && jr.Lifecycle.TriggersState.OnValueChange != nil) { + if jr == nil || jr.Lifecycle == nil { continue } diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 8c5a68d3130..8447d90ad80 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -166,6 +166,14 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Must run after PythonMutator to see development values set by Python mutators. validate.PipelineDevelopmentDeprecated(), + // Copy job_runs on_value_change expressions into triggers_state and resolve them + // there. Must run after PythonMutator: triggers_state is not declared by the PyDABs + // JobRunLifecycle model, so exposing it to Python would fail resource loading. Runs + // before ResolveVolumePathReferencesOnlyResources so that a watched volume_path + // resolves too. + mutator.SnapshotJobRunValueTriggers(), + mutator.ResolveJobRunValueTriggerReferences(), + // Compute resources.volumes.*.volume_path and resolve references to it. Must run after // PythonMutator: volume_path is computed and read-only, not declared by the PyDABs Volume // model, so exposing it to Python would fail resource loading (like "deployment" below). diff --git a/libs/dyn/dynvar/resolve.go b/libs/dyn/dynvar/resolve.go index 99981a2e51b..cc3dfb1857c 100644 --- a/libs/dyn/dynvar/resolve.go +++ b/libs/dyn/dynvar/resolve.go @@ -37,6 +37,12 @@ func Resolve(in dyn.Value, fn Lookup) (out dyn.Value, err error) { return resolver{in: in, fn: fn}.run() } +// ResolveExcept is [Resolve], except that values at paths (relative to in) for +// which skip returns true keep their variable references as they are. +func ResolveExcept(in dyn.Value, fn Lookup, skip func(dyn.Path) bool) (out dyn.Value, err error) { + return resolver{in: in, fn: fn, skip: skip}.run() +} + // ReferenceError is returned for an unresolved variable reference. Suggestions // are carried as data so callers (which can import libs/diag) format them. type ReferenceError struct { @@ -54,8 +60,9 @@ type lookupResult struct { } type resolver struct { //nolint:recvcheck // value receiver for run(), pointer for mutation methods - in dyn.Value - fn Lookup + in dyn.Value + fn Lookup + skip func(dyn.Path) bool refs map[string]Ref resolved map[string]dyn.Value @@ -93,6 +100,9 @@ func (r *resolver) collectVariableReferences() (err error) { // Skip values without variable references. return v, nil } + if r.skip != nil && r.skip(p) { + return v, nil + } r.refs[p.String()] = ref return v, nil diff --git a/libs/dyn/dynvar/resolve_test.go b/libs/dyn/dynvar/resolve_test.go index c4b28e88d3f..7da51722b78 100644 --- a/libs/dyn/dynvar/resolve_test.go +++ b/libs/dyn/dynvar/resolve_test.go @@ -259,6 +259,22 @@ func TestResolveWithSkip(t *testing.T) { assert.Equal(t, "${b} a a ${b}", getByPath(t, out, "f").MustString()) //nolint:dupword } +func TestResolveExcept(t *testing.T) { + in := dyn.V(map[string]dyn.Value{ + "a": dyn.V("a"), + "b": dyn.V("${a}"), + "c": dyn.V("${a}"), + }) + + out, err := dynvar.ResolveExcept(in, dynvar.DefaultLookup(in), func(p dyn.Path) bool { + return p.String() == "c" + }) + require.NoError(t, err) + + assert.Equal(t, "a", getByPath(t, out, "b").MustString()) + assert.Equal(t, "${a}", getByPath(t, out, "c").MustString()) +} + func TestResolveWithSkipEverything(t *testing.T) { in := dyn.V(map[string]dyn.Value{ "a": dyn.V("a"), From 81b4a78ef32b8edc0bd88bd26819fc8696f2b781 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 13:21:10 +0100 Subject: [PATCH 06/12] Reject an on_value_change that resolves to a map or a list The value is stored as a string, so normalization dropped a complex value with a warning and the watch silently never fired. ValidateJobRunTriggers now reports it. The snapshot is written to the dynamic config with the trigger's location, so the normalization warning points at the trigger instead of ":0:0". The old snapshot unit test edited typed config between two applies, which the next apply overwrote from the dynamic config, so it never reached the guard it meant to test; replace it with one for the current behavior. Co-authored-by: Isaac --- .../on_value_change_errors/output.txt | 9 +- .../job_runs/on_value_change_errors/script | 2 +- .../snapshot_job_run_value_triggers.go | 87 ++++++++++++------- .../snapshot_job_run_value_triggers_test.go | 36 +++++--- .../mutator/validate_job_run_triggers.go | 17 ++++ 5 files changed, 104 insertions(+), 47 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt index 4bac8f29791..ede07677287 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt @@ -51,10 +51,13 @@ Workspace: Found 1 error === a complex value is rejected ->>> [CLI] bundle validate +>>> musterr [CLI] bundle validate Warning: expected string, found map at resources.job_runs.my_run.lifecycle.triggers_state.on_value_change['${var.complex}'] - in :0:0 + in databricks.yml:24:38 + +Error: lifecycle.triggers.on_value_change must resolve to a string, number or boolean + in databricks.yml:24:38 Name: job-runs-on-value-change-errors Target: default @@ -62,4 +65,4 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-errors/default -Found 1 warning +Found 1 error and 1 warning diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/script b/acceptance/bundle/resources/job_runs/on_value_change_errors/script index f9ca5adde8a..90d776d498c 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/script @@ -15,4 +15,4 @@ trace musterr $CLI bundle validate title "a complex value is rejected" update_file.py databricks.yml ' triggers: [{on_value_change: "${var.missing}"}]' ' triggers: [{on_value_change: "${var.complex}"}]' -errcode trace $CLI bundle validate +trace musterr $CLI bundle validate diff --git a/bundle/config/mutator/snapshot_job_run_value_triggers.go b/bundle/config/mutator/snapshot_job_run_value_triggers.go index 015aae40f41..0f4b5ec2edb 100644 --- a/bundle/config/mutator/snapshot_job_run_value_triggers.go +++ b/bundle/config/mutator/snapshot_job_run_value_triggers.go @@ -6,8 +6,8 @@ import ( "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" ) type snapshotJobRunValueTriggers struct{} @@ -27,44 +27,71 @@ func (*snapshotJobRunValueTriggers) Name() string { func (*snapshotJobRunValueTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - for name, jr := range b.Config.Resources.JobRuns { - if jr == nil || jr.Lifecycle == nil { - continue - } - - out := make(map[string]string) - for i, trigger := range jr.Lifecycle.Triggers { - if trigger.OnValueChange == nil { + err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { + for name, jr := range b.Config.Resources.JobRuns { + if jr == nil || jr.Lifecycle == nil { continue } - path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_value_change", name, i) - expr := strings.TrimSpace(*trigger.OnValueChange) - if expr == "" { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_value_change must be non-empty when set", - Locations: b.Config.GetLocations(path), - }) - continue + lifecyclePath := dyn.NewPath(dyn.Key("resources"), dyn.Key("job_runs"), dyn.Key(name), dyn.Key("lifecycle")) + watches := make(map[string]dyn.Value) + for i, trigger := range jr.Lifecycle.Triggers { + if trigger.OnValueChange == nil { + continue + } + + path := lifecyclePath.Append(dyn.Key("triggers"), dyn.Index(i), dyn.Key("on_value_change")) + expr := strings.TrimSpace(*trigger.OnValueChange) + if expr == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_value_change must be non-empty when set", + Locations: b.Config.GetLocations(path.String()), + }) + continue + } + if _, ok := watches[expr]; ok { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_value_change expressions must be unique", + Locations: b.Config.GetLocations(path.String()), + }) + continue + } + + // The copy keeps the expression's location, so a diagnostic about its + // resolved value points at the trigger. + v, err := dyn.GetByPath(root, path) + if err != nil { + return dyn.InvalidValue, err + } + watches[expr] = dyn.NewValue(expr, v.Locations()) } - if _, ok := out[expr]; ok { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_value_change expressions must be unique", - Locations: b.Config.GetLocations(path), - }) + if len(watches) == 0 { continue } - out[expr] = expr - } - if len(out) > 0 { - if jr.Lifecycle.TriggersState == nil { - jr.Lifecycle.TriggersState = &resources.JobRunTriggersState{} + statePath := lifecyclePath.Append(dyn.Key("triggers_state")) + state, err := dyn.GetByPath(root, statePath) + if dyn.IsNoSuchKeyError(err) { + state, err = dyn.V(map[string]dyn.Value{}), nil + } + if err != nil { + return dyn.InvalidValue, err + } + state, err = dyn.Set(state, "on_value_change", dyn.V(watches)) + if err != nil { + return dyn.InvalidValue, err + } + root, err = dyn.SetByPath(root, statePath, state) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("snapshot on_value_change of job_runs.%s: %w", name, err) } - jr.Lifecycle.TriggersState.OnValueChange = out } + return root, nil + }) + if err != nil { + diags = diags.Extend(diag.FromErr(err)) } return diags } diff --git a/bundle/config/mutator/snapshot_job_run_value_triggers_test.go b/bundle/config/mutator/snapshot_job_run_value_triggers_test.go index 7d7a7cd5bf5..f7f28e6b7c5 100644 --- a/bundle/config/mutator/snapshot_job_run_value_triggers_test.go +++ b/bundle/config/mutator/snapshot_job_run_value_triggers_test.go @@ -7,24 +7,34 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestSnapshotJobRunValueTriggersIsStableAfterInterpolation(t *testing.T) { - expr := "${var.value}" - resolved := "resolved" - jobRun := &resources.JobRun{ - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{{OnValueChange: &expr}}, - }, - } +func TestSnapshotJobRunValueTriggers(t *testing.T) { + padded := " ${var.value} " + duplicate := "${var.value}" b := &bundle.Bundle{Config: config.Root{ - Resources: config.Resources{JobRuns: map[string]*resources.JobRun{"run": jobRun}}, + Resources: config.Resources{JobRuns: map[string]*resources.JobRun{"run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{{OnValueChange: &padded}, {OnValueChange: &duplicate}}, + }, + }}}, }} + bundletest.SetLocation(b, "resources.job_runs.run", []diag.Location{{File: "databricks.yml", Line: 1, Column: 1}}) - assert.Empty(t, bundle.Apply(t.Context(), b, mutator.SnapshotJobRunValueTriggers())) - b.Config.Resources.JobRuns["run"].Lifecycle.Triggers[0].OnValueChange = &resolved - assert.Empty(t, bundle.Apply(t.Context(), b, mutator.SnapshotJobRunValueTriggers())) + diags := bundle.Apply(t.Context(), b, mutator.SnapshotJobRunValueTriggers()) + require.Len(t, diags, 1) + assert.Equal(t, "lifecycle.triggers.on_value_change expressions must be unique", diags[0].Summary) - assert.Equal(t, map[string]string{expr: expr}, b.Config.Resources.JobRuns["run"].Lifecycle.TriggersState.OnValueChange) + assert.Equal(t, map[string]string{"${var.value}": "${var.value}"}, b.Config.Resources.JobRuns["run"].Lifecycle.TriggersState.OnValueChange) + // The trigger is untouched: its expression stays the key on every deploy. + assert.Equal(t, padded, *b.Config.Resources.JobRuns["run"].Lifecycle.Triggers[0].OnValueChange) + + watch, err := dyn.GetByPath(b.Config.Value(), dyn.MustPathFromString("resources.job_runs.run.lifecycle.triggers_state.on_value_change").Append(dyn.Key("${var.value}"))) + require.NoError(t, err) + assert.Equal(t, []diag.Location{{File: "databricks.yml", Line: 1, Column: 1}}, watch.Locations()) } diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index cb12587a306..b8e0d001c38 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -26,6 +26,10 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D if jr == nil || jr.Lifecycle == nil { continue } + var watched map[string]string + if jr.Lifecycle.TriggersState != nil { + watched = jr.Lifecycle.TriggersState.OnValueChange + } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) armed := t.ArmedCount() @@ -65,6 +69,19 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D _, patternDiags := validateFileTriggerPattern(b, path+".on_file_change", *t.OnFileChange) diags = diags.Extend(patternDiags) } + if t.OnValueChange != nil { + // SnapshotJobRunValueTriggers reports empty and duplicate expressions. Any + // other expression is in triggers_state unless it resolved to a map or a + // list, which normalization dropped with a warning. + expr := strings.TrimSpace(*t.OnValueChange) + if _, ok := watched[expr]; expr != "" && !ok { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_value_change must resolve to a string, number or boolean", + Locations: b.Config.GetLocations(path + ".on_value_change"), + }) + } + } } } return diags From 4096bef39e024fa8081c0161f89bdee01dfa29f3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 13:23:40 +0100 Subject: [PATCH 07/12] Document that re-adding an on_value_change compares to its last value Removing a watch keeps its last value in state, the same as on_file_change, so re-adding it with an unchanged value does not re-fire the run. The field description said any added watch re-fires, and the test step that shows the opposite was titled "re-fires". Describe the rule as a comparison with the value recorded when the run last fired, and fix the title. Co-authored-by: Isaac --- .../bundle/resources/job_runs/on_value_change/output.txt | 2 +- acceptance/bundle/resources/job_runs/on_value_change/script | 2 +- bundle/direct/dresources/job_run.go | 4 ++-- bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- python/databricks/bundles/job_runs/_models/job_run_trigger.py | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_value_change/output.txt b/acceptance/bundle/resources/job_runs/on_value_change/output.txt index 933acd095a5..63461bbb830 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change/output.txt @@ -101,7 +101,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now -=== re-adding the same on_value_change after the saved plan re-fires +=== re-adding a removed on_value_change with an unchanged value does not re-fire >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/job_runs/on_value_change/script b/acceptance/bundle/resources/job_runs/on_value_change/script index 46952327e26..3aa2fe4b840 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/script +++ b/acceptance/bundle/resources/job_runs/on_value_change/script @@ -39,7 +39,7 @@ $CLI bundle plan -o json > tmp.plan.json trace $CLI bundle deploy --plan tmp.plan.json trace print_requests.py //jobs/run-now -title "re-adding the same on_value_change after the saved plan re-fires" +title "re-adding a removed on_value_change with an unchanged value does not re-fire" update_file.py databricks.yml ' - on_value_change: " ${var.stamp} " ' ' - on_value_change: " ${var.stamp} " - on_value_change: ${var.other} diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 51d0af16e88..85f75a93764 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -392,8 +392,8 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // stopped without succeeding keeps its recreate. A SKIPPED run reports no // result_state either, so the lifecycle state is what tells the two apart. // Removing a trigger drops the change and leaves the last fingerprint in state, -// so re-adding the trigger only re-fires when the watched files changed -// meanwhile. All other trigger changes recreate the run. +// so re-adding the trigger only re-fires when the watched files or value +// changed meanwhile. All other trigger changes recreate the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { switch path.String() { case "lifecycle": diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 07023f7148f..1a2a72d114f 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1135,7 +1135,7 @@ resources: Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. "on_value_change": "description": |- - Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Removing a watch disarms it without re-firing the run. + Interpolated expression. Re-fire the run when its resolved value differs from the value recorded when the run last fired, or when no value was recorded for it. Resource references in the expression become deployment dependencies. Removing a watch does not re-fire the run. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 00c29b156f7..a9b89268b9d 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1710,7 +1710,7 @@ "$ref": "#/$defs/string" }, "on_value_change": { - "description": "Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Removing a watch disarms it without re-firing the run.", + "description": "Interpolated expression. Re-fire the run when its resolved value differs from the value recorded when the run last fired, or when no value was recorded for it. Resource references in the expression become deployment dependencies. Removing a watch does not re-fire the run.", "$ref": "#/$defs/string" } }, diff --git a/python/databricks/bundles/job_runs/_models/job_run_trigger.py b/python/databricks/bundles/job_runs/_models/job_run_trigger.py index 84c12cf7a24..a25a0e39959 100644 --- a/python/databricks/bundles/job_runs/_models/job_run_trigger.py +++ b/python/databricks/bundles/job_runs/_models/job_run_trigger.py @@ -27,7 +27,7 @@ class JobRunTrigger: on_value_change: VariableOrOptional[str] = None """ - Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Removing a watch disarms it without re-firing the run. + Interpolated expression. Re-fire the run when its resolved value differs from the value recorded when the run last fired, or when no value was recorded for it. Resource references in the expression become deployment dependencies. Removing a watch does not re-fire the run. """ @classmethod @@ -53,7 +53,7 @@ class JobRunTriggerDict(TypedDict, total=False): on_value_change: VariableOrOptional[str] """ - Interpolated expression. Re-fire the run when the resolved value changes or when a watch is added. Resource references in the expression become deployment dependencies. Removing a watch disarms it without re-firing the run. + Interpolated expression. Re-fire the run when its resolved value differs from the value recorded when the run last fired, or when no value was recorded for it. Resource references in the expression become deployment dependencies. Removing a watch does not re-fire the run. """ From 019788e390c045bf8826eb7bc7f443ca590bc330 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 13:26:30 +0100 Subject: [PATCH 08/12] Drop job_run PrepareInputConfig and the triggers_state clone The planner's extractReferences already registers the ${resources.*} references in lifecycle.triggers_state.on_value_change from the dynamic config, so PrepareInputConfig added the same refs a second time; the README reserves the hook for sub-resources. The maps.Clone in PrepareState protected entries from fingerprinting that no longer exists. Co-authored-by: Isaac --- bundle/direct/dresources/job_run.go | 25 ------------------------ bundle/direct/dresources/job_run_test.go | 3 --- 2 files changed, 28 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 85f75a93764..d122b688839 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "maps" "slices" "strconv" "strings" @@ -15,10 +14,8 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -29,10 +26,6 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour -// on_value_change fingerprints are keyed by the watched expression; the planner -// resolves references at this path, so it is kept as a parsed node. -var jobRunOnValueChangePath = structpath.MustParsePath("lifecycle.triggers_state.on_value_change") - // JobRunLifecycleState is the local-only trigger fingerprint. type JobRunLifecycleState struct { TriggersState *resources.JobRunTriggersState `json:"triggers_state,omitempty"` @@ -103,8 +96,6 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { var ts resources.JobRunTriggersState if input.Lifecycle != nil && input.Lifecycle.TriggersState != nil { ts = *input.Lifecycle.TriggersState - // Fingerprinting below rewrites value-trigger entries. - ts.OnValueChange = maps.Clone(ts.OnValueChange) } if input.HasOnBundleDeploy() { ts.OnBundleDeploy = uuid.New().String() @@ -115,22 +106,6 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { return state } -// PrepareInputConfig maps unresolved watches to state paths so the deploy graph -// tracks their resource references. -func (*ResourceJobRun) PrepareInputConfig(input *resources.JobRun, _ string) (*structvar.StructVar, error) { - refs := map[string]string{} - if input.Lifecycle == nil || input.Lifecycle.TriggersState == nil { - return &structvar.StructVar{Value: input, Refs: refs}, nil - } - for expr, value := range input.Lifecycle.TriggersState.OnValueChange { - if !dynvar.ContainsVariableReference(value) { - continue - } - refs[structpath.NewBracketString(jobRunOnValueChangePath, expr).String()] = value - } - return &structvar.StructVar{Value: input, Refs: refs}, nil -} - // makeJobRunRemote maps the GetRun response into the RunNow-shaped remote: GET // nests the params under overriding_parameters and returns job_parameters as a // list, so both are flattened back into RunNow. diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 4d3a25dbf2a..3b6735549b8 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -292,9 +292,6 @@ func TestJobRunPrepareStateCopiesResolvedTriggers(t *testing.T) { assert.Equal(t, map[string]string{ "${var.watched}": "resolved", }, state.Lifecycle.TriggersState.OnValueChange) - assert.Equal(t, map[string]string{ - "${var.watched}": "resolved", - }, triggers.OnValueChange) assert.NotEmpty(t, state.Lifecycle.TriggersState.OnBundleDeploy) assert.Empty(t, triggers.OnBundleDeploy) } From b54a9a8a710b7505a43412e3426a2235c5d640c6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 13:27:56 +0100 Subject: [PATCH 09/12] Sort job_runs in SnapshotJobRunValueTriggers, drop unused repl Iterating the job_runs map made the order of diagnostics from several job_runs random; ResolveJobRunFileTriggers sorts for the same reason. The [VALUE_HASH] replacement in on_value_change_ref was left over from when values were stored as hashes and matches nothing. Co-authored-by: Isaac --- .../bundle/resources/job_runs/on_value_change_ref/test.toml | 6 ------ bundle/config/mutator/snapshot_job_run_value_triggers.go | 6 +++++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml b/acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml index 803d3f4a8b9..a698f7a65d1 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml @@ -3,9 +3,3 @@ EnvMatrix.READPLAN = ["", "1"] Ignore = ["tmp.plan.json"] - -[[Repls]] -Old = '[0-9a-f]{64}' -New = '[VALUE_HASH]' -Order = 1 -Distinct = true diff --git a/bundle/config/mutator/snapshot_job_run_value_triggers.go b/bundle/config/mutator/snapshot_job_run_value_triggers.go index 0f4b5ec2edb..a35149d8c6d 100644 --- a/bundle/config/mutator/snapshot_job_run_value_triggers.go +++ b/bundle/config/mutator/snapshot_job_run_value_triggers.go @@ -3,6 +3,8 @@ package mutator import ( "context" "fmt" + "maps" + "slices" "strings" "github.com/databricks/cli/bundle" @@ -28,7 +30,9 @@ func (*snapshotJobRunValueTriggers) Name() string { func (*snapshotJobRunValueTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - for name, jr := range b.Config.Resources.JobRuns { + // Sorted so diagnostics from several job_runs come out in a stable order. + for _, name := range slices.Sorted(maps.Keys(b.Config.Resources.JobRuns)) { + jr := b.Config.Resources.JobRuns[name] if jr == nil || jr.Lifecycle == nil { continue } From 89dee8fffc1c22cff9b154a823aab129055a76a5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 15:12:40 +0100 Subject: [PATCH 10/12] Interpolate on_value_change normally; record expressions on the Bundle Replace the resolver change from the previous commits: on_value_change no longer stays unresolved (dynvar.ResolveExcept and keepUnresolved are gone), so the trigger shows its resolved value like any other field. SnapshotJobRunValueTriggers records the expressions on the Bundle before variable resolution (as RootPathIsNameTargetScoped does), and after PythonMutator ResolveJobRunValueTriggers stores the resolved trigger values in triggers_state under them. Python never sees triggers_state. Values are paired with expressions by trigger position, so a Python mutator that adds or removes triggers of a job_run with on_value_change gets an error. Co-authored-by: Isaac --- .../job_runs-on_value_change/databricks.yml | 2 +- .../job_runs-on_value_change/mutators.py | 5 + .../job_runs-on_value_change/output.txt | 50 +++--- .../python/job_runs-on_value_change/script | 11 +- .../on_value_change_errors/output.txt | 4 +- bundle/bundle.go | 5 + .../mutator/resolve_variable_references.go | 36 +--- .../process_static_resources.go | 3 + .../resourcemutator/resource_mutator.go | 3 + .../snapshot_job_run_value_triggers.go | 154 +++++++++++------- .../snapshot_job_run_value_triggers_test.go | 31 ++-- .../mutator/validate_job_run_triggers.go | 23 +-- bundle/phases/initialize.go | 11 +- libs/dyn/dynvar/resolve.go | 14 +- libs/dyn/dynvar/resolve_test.go | 16 -- 15 files changed, 175 insertions(+), 193 deletions(-) diff --git a/acceptance/bundle/python/job_runs-on_value_change/databricks.yml b/acceptance/bundle/python/job_runs-on_value_change/databricks.yml index dca8ab672c0..acef9e1708a 100644 --- a/acceptance/bundle/python/job_runs-on_value_change/databricks.yml +++ b/acceptance/bundle/python/job_runs-on_value_change/databricks.yml @@ -13,7 +13,7 @@ python: resources: - "resources:load_resources" mutators: - - "mutators:watch_y" + - "mutators:update_job_run" resources: jobs: diff --git a/acceptance/bundle/python/job_runs-on_value_change/mutators.py b/acceptance/bundle/python/job_runs-on_value_change/mutators.py index 15ef3dc2f58..91c20861407 100644 --- a/acceptance/bundle/python/job_runs-on_value_change/mutators.py +++ b/acceptance/bundle/python/job_runs-on_value_change/mutators.py @@ -4,6 +4,11 @@ from databricks.bundles.job_runs import JobRun, JobRunLifecycle, JobRunTrigger +@job_run_mutator +def update_job_run(run: JobRun) -> JobRun: + return replace(run, job_parameters={"source": "mutator"}) + + @job_run_mutator def watch_y(run: JobRun) -> JobRun: lifecycle = run.lifecycle or JobRunLifecycle() diff --git a/acceptance/bundle/python/job_runs-on_value_change/output.txt b/acceptance/bundle/python/job_runs-on_value_change/output.txt index ee296d0028f..deec2a3b5af 100644 --- a/acceptance/bundle/python/job_runs-on_value_change/output.txt +++ b/acceptance/bundle/python/job_runs-on_value_change/output.txt @@ -1,41 +1,39 @@ -=== validate keeps a watch from YAML, from Python and from a Python mutator +=== validate resolves a watch from YAML and from Python, with a mutator applied >>> uv run [UV_ARGS] -q [CLI] bundle validate --output json { "py_run": { "job_id": "${resources.jobs.my_job.id}", + "job_parameters": { + "source": "mutator" + }, "lifecycle": { "triggers": [ { - "on_value_change": "${var.x}" - }, - { - "on_value_change": "${var.y}" + "on_value_change": "x1" } ], "triggers_state": { "on_value_change": { - "${var.x}": "x1", - "${var.y}": "y1" + "${var.x}": "x1" } } } }, "yaml_run": { "job_id": "${resources.jobs.my_job.id}", + "job_parameters": { + "source": "mutator" + }, "lifecycle": { "triggers": [ { - "on_value_change": "${var.x}" - }, - { - "on_value_change": "${var.y}" + "on_value_change": "x1" } ], "triggers_state": { "on_value_change": { - "${var.x}": "x1", - "${var.y}": "y1" + "${var.x}": "x1" } } } @@ -55,7 +53,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> uv run [UV_ARGS] -q [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged -=== changing the variable watched in YAML and Python re-fires both runs +=== changing the watched variable re-fires both runs >>> uv run [UV_ARGS] -q [CLI] bundle plan recreate job_runs.py_run recreate job_runs.yaml_run @@ -69,16 +67,18 @@ Recreated job_runs.yaml_run Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 2 deleted, 1 unchanged -=== changing the variable the Python mutator watches re-fires both runs ->>> uv run [UV_ARGS] -q [CLI] bundle plan -recreate job_runs.py_run -recreate job_runs.yaml_run +=== a Python mutator cannot add a trigger to a job_run that sets on_value_change +>>> musterr uv run [UV_ARGS] -q [CLI] bundle validate +Error: Python mutators cannot add or remove lifecycle.triggers of a job_run that sets on_value_change + in resources.py:7:1 -Plan: 2 to add, 0 to change, 2 to delete, 1 unchanged +Error: Python mutators cannot add or remove lifecycle.triggers of a job_run that sets on_value_change + in databricks.yml:32:11 ->>> uv run [UV_ARGS] -q [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Recreated job_runs.py_run -Recreated job_runs.yaml_run -Files: 0 uploaded, 0 deleted -Resources: 2 created, 0 changed, 2 deleted, 1 unchanged +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 2 errors diff --git a/acceptance/bundle/python/job_runs-on_value_change/script b/acceptance/bundle/python/job_runs-on_value_change/script index 94bad42f2c3..e7dbd54c474 100644 --- a/acceptance/bundle/python/job_runs-on_value_change/script +++ b/acceptance/bundle/python/job_runs-on_value_change/script @@ -1,4 +1,4 @@ -title "validate keeps a watch from YAML, from Python and from a Python mutator" +title "validate resolves a watch from YAML and from Python, with a mutator applied" trace uv run $UV_ARGS -q $CLI bundle validate --output json | jq .resources.job_runs title "first deploy runs both" @@ -7,14 +7,13 @@ trace uv run $UV_ARGS -q $CLI bundle deploy title "redeploy with unchanged values plans nothing" trace uv run $UV_ARGS -q $CLI bundle plan -title "changing the variable watched in YAML and Python re-fires both runs" +title "changing the watched variable re-fires both runs" update_file.py databricks.yml "default: x1" "default: x2" trace uv run $UV_ARGS -q $CLI bundle plan trace uv run $UV_ARGS -q $CLI bundle deploy -title "changing the variable the Python mutator watches re-fires both runs" -update_file.py databricks.yml "default: y1" "default: y2" -trace uv run $UV_ARGS -q $CLI bundle plan -trace uv run $UV_ARGS -q $CLI bundle deploy +title "a Python mutator cannot add a trigger to a job_run that sets on_value_change" +update_file.py databricks.yml '"mutators:update_job_run"' '"mutators:watch_y"' +trace musterr uv run $UV_ARGS -q $CLI bundle validate rm -fr .databricks __pycache__ diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt index ede07677287..46ada1c6590 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt @@ -53,11 +53,11 @@ Found 1 error === a complex value is rejected >>> musterr [CLI] bundle validate Warning: expected string, found map - at resources.job_runs.my_run.lifecycle.triggers_state.on_value_change['${var.complex}'] + at resources.job_runs.my_run.lifecycle.triggers[0].on_value_change in databricks.yml:24:38 Error: lifecycle.triggers.on_value_change must resolve to a string, number or boolean - in databricks.yml:24:38 + in databricks.yml:24:20 Name: job-runs-on-value-change-errors Target: default diff --git a/bundle/bundle.go b/bundle/bundle.go index 6762b38b91e..91b1c5e4e57 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -161,6 +161,11 @@ type Bundle struct { // to end in those two segments does not count. RootPathIsNameTargetScoped bool + // JobRunValueTriggers holds, per job_run, the on_value_change expression of each + // lifecycle.triggers entry ("" for other entries), recorded before variable + // resolution. ResolveJobRunValueTriggers keys the resolved values by them. + JobRunValueTriggers map[string][]string + // Metadata about the bundle deployment. This is the interface Databricks services // rely on to integrate with bundles when they need additional information about // a bundle deployment. diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index 1a85396d064..a5e0d3bf086 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -48,21 +48,6 @@ var defaultPrefixes = []string{ var artifactPath = dyn.MustPathFromString("artifacts") -// keepUnresolved holds the locations whose references are never resolved in place. -// -// A job_runs on_value_change expression is the key its resolved value is recorded -// under, so it has to survive until SnapshotJobRunValueTriggers copies it into -// triggers_state after PythonMutator; ResolveJobRunValueTriggerReferences then -// resolves the copy. -var keepUnresolved = func() *dyn.TrieNode { - trie := &dyn.TrieNode{} - err := trie.Insert(dyn.MustPatternFromString("resources.job_runs.*.lifecycle.triggers[*].on_value_change")) - if err != nil { - panic(err) - } - return trie -}() - type resolveVariableReferences struct { prefixes []string pattern dyn.Pattern @@ -118,18 +103,6 @@ func ResolveVariableReferencesInLookup() bundle.Mutator { } } -// ResolveJobRunValueTriggerReferences resolves the on_value_change expressions that -// SnapshotJobRunValueTriggers copied into resources.job_runs.*.lifecycle.triggers_state. -func ResolveJobRunValueTriggerReferences() bundle.Mutator { - return &resolveVariableReferences{ - prefixes: defaultPrefixes, - lookupFn: lookup, - extraRounds: maxResolutionRounds - 1, - pattern: dyn.MustPatternFromString("resources.job_runs.*.lifecycle.triggers_state.on_value_change"), - includeResources: true, - } -} - // ResolveVolumePathReferencesOnlyResources resolves only references to resources.volumes.*.volume_path. func ResolveVolumePathReferencesOnlyResources() bundle.Mutator { return &resolveVariableReferences{ @@ -288,13 +261,8 @@ func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn // If the pattern is nil, we resolve references in the entire configuration. root, err := dyn.MapByPattern(root, m.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - kept := func(rel dyn.Path) bool { - _, ok := keepUnresolved.SearchPath(slices.Concat(p, rel)) - return ok - } - // Resolve variable references in all values. - return dynvar.ResolveExcept(v, func(path dyn.Path) (dyn.Value, error) { + return dynvar.Resolve(v, func(path dyn.Path) (dyn.Value, error) { // Rewrite the shorthand path ${var.foo} into ${variables.foo.value}. if path.HasPrefix(varPath) { newPath := dyn.NewPath( @@ -329,7 +297,7 @@ func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn } return dyn.InvalidValue, dynvar.ErrSkipResolution - }, kept) + }) }) if err != nil { return dyn.InvalidValue, err diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index 06ec593f092..920bbbfc6f6 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -41,6 +41,9 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia bundle.ApplySeqContext( ctx, b, + // Record on_value_change expressions before variable resolution replaces them. + mutator.SnapshotJobRunValueTriggers(), + // Reads (dynamic): * (strings) (searches for variable references in string values) // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index e00444cc773..82c5422bc80 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -139,6 +139,9 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { validate.SingleNodeCluster(), + // Record on_value_change expressions before variable resolution replaces them. + mutator.SnapshotJobRunValueTriggers(), + // Reads (dynamic): * (strings) (searches for variable references in string values) // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes diff --git a/bundle/config/mutator/snapshot_job_run_value_triggers.go b/bundle/config/mutator/snapshot_job_run_value_triggers.go index a35149d8c6d..f53f1a7a2bb 100644 --- a/bundle/config/mutator/snapshot_job_run_value_triggers.go +++ b/bundle/config/mutator/snapshot_job_run_value_triggers.go @@ -8,17 +8,16 @@ import ( "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" ) type snapshotJobRunValueTriggers struct{} -// SnapshotJobRunValueTriggers copies each on_value_change expression into -// lifecycle.triggers_state, keyed by the trimmed expression, for -// ResolveJobRunValueTriggerReferences to resolve. The expression itself is never -// resolved in place (see keepUnresolved), so the key stays stable when the value -// changes. +// SnapshotJobRunValueTriggers records each job_run's on_value_change expressions in +// b.JobRunValueTriggers before variable resolution, so that ResolveJobRunValueTriggers +// can key the resolved values by the expression. A job_run that is already recorded is +// skipped: when a Python mutator updates it, its triggers are already resolved. func SnapshotJobRunValueTriggers() bundle.Mutator { return &snapshotJobRunValueTriggers{} } @@ -29,73 +28,104 @@ func (*snapshotJobRunValueTriggers) Name() string { func (*snapshotJobRunValueTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - // Sorted so diagnostics from several job_runs come out in a stable order. - for _, name := range slices.Sorted(maps.Keys(b.Config.Resources.JobRuns)) { - jr := b.Config.Resources.JobRuns[name] - if jr == nil || jr.Lifecycle == nil { + // Sorted so diagnostics from several job_runs come out in a stable order. + for _, name := range slices.Sorted(maps.Keys(b.Config.Resources.JobRuns)) { + jr := b.Config.Resources.JobRuns[name] + if jr == nil || jr.Lifecycle == nil || b.JobRunValueTriggers[name] != nil { + continue + } + + exprs := make([]string, len(jr.Lifecycle.Triggers)) + seen := make(map[string]bool) + for i, trigger := range jr.Lifecycle.Triggers { + if trigger.OnValueChange == nil { continue } - lifecyclePath := dyn.NewPath(dyn.Key("resources"), dyn.Key("job_runs"), dyn.Key(name), dyn.Key("lifecycle")) - watches := make(map[string]dyn.Value) - for i, trigger := range jr.Lifecycle.Triggers { - if trigger.OnValueChange == nil { - continue - } - - path := lifecyclePath.Append(dyn.Key("triggers"), dyn.Index(i), dyn.Key("on_value_change")) - expr := strings.TrimSpace(*trigger.OnValueChange) - if expr == "" { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_value_change must be non-empty when set", - Locations: b.Config.GetLocations(path.String()), - }) - continue - } - if _, ok := watches[expr]; ok { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_value_change expressions must be unique", - Locations: b.Config.GetLocations(path.String()), - }) - continue - } - - // The copy keeps the expression's location, so a diagnostic about its - // resolved value points at the trigger. - v, err := dyn.GetByPath(root, path) - if err != nil { - return dyn.InvalidValue, err - } - watches[expr] = dyn.NewValue(expr, v.Locations()) + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_value_change", name, i) + expr := strings.TrimSpace(*trigger.OnValueChange) + if expr == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_value_change must be non-empty when set", + Locations: b.Config.GetLocations(path), + }) + continue } - if len(watches) == 0 { + if seen[expr] { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_value_change expressions must be unique", + Locations: b.Config.GetLocations(path), + }) continue } + seen[expr] = true + exprs[i] = expr + } - statePath := lifecyclePath.Append(dyn.Key("triggers_state")) - state, err := dyn.GetByPath(root, statePath) - if dyn.IsNoSuchKeyError(err) { - state, err = dyn.V(map[string]dyn.Value{}), nil + if len(seen) > 0 { + if b.JobRunValueTriggers == nil { + b.JobRunValueTriggers = make(map[string][]string) } - if err != nil { - return dyn.InvalidValue, err - } - state, err = dyn.Set(state, "on_value_change", dyn.V(watches)) - if err != nil { - return dyn.InvalidValue, err + b.JobRunValueTriggers[name] = exprs + } + } + return diags +} + +type resolveJobRunValueTriggers struct{} + +// ResolveJobRunValueTriggers stores each job_run's resolved on_value_change values in +// lifecycle.triggers_state, keyed by the expressions SnapshotJobRunValueTriggers +// recorded. The values are paired with the expressions by trigger position, so a +// Python mutator must not add or remove triggers of a job_run that sets on_value_change. +func ResolveJobRunValueTriggers() bundle.Mutator { + return &resolveJobRunValueTriggers{} +} + +func (*resolveJobRunValueTriggers) Name() string { + return "ResolveJobRunValueTriggers" +} + +func (*resolveJobRunValueTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + for _, name := range slices.Sorted(maps.Keys(b.JobRunValueTriggers)) { + exprs := b.JobRunValueTriggers[name] + jr := b.Config.Resources.JobRuns[name] + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) + if jr.Lifecycle == nil || len(jr.Lifecycle.Triggers) != len(exprs) { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "Python mutators cannot add or remove lifecycle.triggers of a job_run that sets on_value_change", + Locations: b.Config.GetLocations(path), + }) + continue + } + + values := make(map[string]string) + for i, expr := range exprs { + if expr == "" { + continue } - root, err = dyn.SetByPath(root, statePath, state) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("snapshot on_value_change of job_runs.%s: %w", name, err) + value := jr.Lifecycle.Triggers[i].OnValueChange + if value == nil { + // Normalization dropped a value that is a map or a list, with a warning. + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_value_change must resolve to a string, number or boolean", + Locations: b.Config.GetLocations(fmt.Sprintf("%s[%d]", path, i)), + }) + continue } + // Trimmed like the expression, so padding the expression does not re-fire. + values[expr] = strings.TrimSpace(*value) + } + + if jr.Lifecycle.TriggersState == nil { + jr.Lifecycle.TriggersState = &resources.JobRunTriggersState{} } - return root, nil - }) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) + jr.Lifecycle.TriggersState.OnValueChange = values } return diags } diff --git a/bundle/config/mutator/snapshot_job_run_value_triggers_test.go b/bundle/config/mutator/snapshot_job_run_value_triggers_test.go index f7f28e6b7c5..99e597a94eb 100644 --- a/bundle/config/mutator/snapshot_job_run_value_triggers_test.go +++ b/bundle/config/mutator/snapshot_job_run_value_triggers_test.go @@ -7,34 +7,43 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestSnapshotJobRunValueTriggers(t *testing.T) { + enabled := true padded := " ${var.value} " duplicate := "${var.value}" b := &bundle.Bundle{Config: config.Root{ Resources: config.Resources{JobRuns: map[string]*resources.JobRun{"run": { Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{{OnValueChange: &padded}, {OnValueChange: &duplicate}}, + Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &enabled}, {OnValueChange: &padded}, {OnValueChange: &duplicate}}, }, }}}, }} - bundletest.SetLocation(b, "resources.job_runs.run", []diag.Location{{File: "databricks.yml", Line: 1, Column: 1}}) diags := bundle.Apply(t.Context(), b, mutator.SnapshotJobRunValueTriggers()) require.Len(t, diags, 1) assert.Equal(t, "lifecycle.triggers.on_value_change expressions must be unique", diags[0].Summary) + assert.Equal(t, map[string][]string{"run": {"", "${var.value}", ""}}, b.JobRunValueTriggers) +} - assert.Equal(t, map[string]string{"${var.value}": "${var.value}"}, b.Config.Resources.JobRuns["run"].Lifecycle.TriggersState.OnValueChange) - // The trigger is untouched: its expression stays the key on every deploy. - assert.Equal(t, padded, *b.Config.Resources.JobRuns["run"].Lifecycle.Triggers[0].OnValueChange) +func TestResolveJobRunValueTriggers(t *testing.T) { + resolved := " resolved " + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{JobRuns: map[string]*resources.JobRun{"run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{{OnValueChange: &resolved}, {}}, + }, + }}}, + }, + JobRunValueTriggers: map[string][]string{"run": {"${var.value}", "${var.complex}"}}, + } - watch, err := dyn.GetByPath(b.Config.Value(), dyn.MustPathFromString("resources.job_runs.run.lifecycle.triggers_state.on_value_change").Append(dyn.Key("${var.value}"))) - require.NoError(t, err) - assert.Equal(t, []diag.Location{{File: "databricks.yml", Line: 1, Column: 1}}, watch.Locations()) + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunValueTriggers()) + require.Len(t, diags, 1) + assert.Equal(t, "lifecycle.triggers.on_value_change must resolve to a string, number or boolean", diags[0].Summary) + assert.Equal(t, map[string]string{"${var.value}": "resolved"}, b.Config.Resources.JobRuns["run"].Lifecycle.TriggersState.OnValueChange) } diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index b8e0d001c38..106352fe596 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -26,13 +26,15 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D if jr == nil || jr.Lifecycle == nil { continue } - var watched map[string]string - if jr.Lifecycle.TriggersState != nil { - watched = jr.Lifecycle.TriggersState.OnValueChange - } + recorded := b.JobRunValueTriggers[name] for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) armed := t.ArmedCount() + // DropEmptyStrings removes an on_value_change whose value is "", which is a + // valid value to watch; the expression is still recorded. + if t.OnValueChange == nil && i < len(recorded) && recorded[i] != "" { + armed++ + } if armed == 0 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, @@ -69,19 +71,6 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D _, patternDiags := validateFileTriggerPattern(b, path+".on_file_change", *t.OnFileChange) diags = diags.Extend(patternDiags) } - if t.OnValueChange != nil { - // SnapshotJobRunValueTriggers reports empty and duplicate expressions. Any - // other expression is in triggers_state unless it resolved to a map or a - // list, which normalization dropped with a warning. - expr := strings.TrimSpace(*t.OnValueChange) - if _, ok := watched[expr]; expr != "" && !ok { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_value_change must resolve to a string, number or boolean", - Locations: b.Config.GetLocations(path + ".on_value_change"), - }) - } - } } } return diags diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 8447d90ad80..823408061db 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -166,13 +166,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Must run after PythonMutator to see development values set by Python mutators. validate.PipelineDevelopmentDeprecated(), - // Copy job_runs on_value_change expressions into triggers_state and resolve them - // there. Must run after PythonMutator: triggers_state is not declared by the PyDABs - // JobRunLifecycle model, so exposing it to Python would fail resource loading. Runs - // before ResolveVolumePathReferencesOnlyResources so that a watched volume_path - // resolves too. - mutator.SnapshotJobRunValueTriggers(), - mutator.ResolveJobRunValueTriggerReferences(), + // Store job_runs on_value_change values in triggers_state. Must run after + // PythonMutator: triggers_state is not declared by the PyDABs JobRunLifecycle + // model, so exposing it to Python would fail resource loading. + mutator.ResolveJobRunValueTriggers(), // Compute resources.volumes.*.volume_path and resolve references to it. Must run after // PythonMutator: volume_path is computed and read-only, not declared by the PyDABs Volume diff --git a/libs/dyn/dynvar/resolve.go b/libs/dyn/dynvar/resolve.go index cc3dfb1857c..99981a2e51b 100644 --- a/libs/dyn/dynvar/resolve.go +++ b/libs/dyn/dynvar/resolve.go @@ -37,12 +37,6 @@ func Resolve(in dyn.Value, fn Lookup) (out dyn.Value, err error) { return resolver{in: in, fn: fn}.run() } -// ResolveExcept is [Resolve], except that values at paths (relative to in) for -// which skip returns true keep their variable references as they are. -func ResolveExcept(in dyn.Value, fn Lookup, skip func(dyn.Path) bool) (out dyn.Value, err error) { - return resolver{in: in, fn: fn, skip: skip}.run() -} - // ReferenceError is returned for an unresolved variable reference. Suggestions // are carried as data so callers (which can import libs/diag) format them. type ReferenceError struct { @@ -60,9 +54,8 @@ type lookupResult struct { } type resolver struct { //nolint:recvcheck // value receiver for run(), pointer for mutation methods - in dyn.Value - fn Lookup - skip func(dyn.Path) bool + in dyn.Value + fn Lookup refs map[string]Ref resolved map[string]dyn.Value @@ -100,9 +93,6 @@ func (r *resolver) collectVariableReferences() (err error) { // Skip values without variable references. return v, nil } - if r.skip != nil && r.skip(p) { - return v, nil - } r.refs[p.String()] = ref return v, nil diff --git a/libs/dyn/dynvar/resolve_test.go b/libs/dyn/dynvar/resolve_test.go index 7da51722b78..c4b28e88d3f 100644 --- a/libs/dyn/dynvar/resolve_test.go +++ b/libs/dyn/dynvar/resolve_test.go @@ -259,22 +259,6 @@ func TestResolveWithSkip(t *testing.T) { assert.Equal(t, "${b} a a ${b}", getByPath(t, out, "f").MustString()) //nolint:dupword } -func TestResolveExcept(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${a}"), - }) - - out, err := dynvar.ResolveExcept(in, dynvar.DefaultLookup(in), func(p dyn.Path) bool { - return p.String() == "c" - }) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "${a}", getByPath(t, out, "c").MustString()) -} - func TestResolveWithSkipEverything(t *testing.T) { in := dyn.V(map[string]dyn.Value{ "a": dyn.V("a"), From 9aa85a80d60b3ea2983dabb7069140e1820b968b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 15:47:55 +0100 Subject: [PATCH 11/12] Compare on_value_change values by position Replace the expression-keyed state with the interpolated values in trigger order. ResolveJobRunValueTriggers copies them into triggers_state after PythonMutator; nothing is recorded before interpolation, so the snapshot mutator, the Bundle field and the Python mutator limitation are gone, and duplicate or blank expressions are no longer errors. The planner compares the values per position: a changed or added value re-fires the run, and a list that keeps the leading values of the old one (watches removed from the end) does not. Removing or reordering any other watch shifts the values after it, which re-fires. ValidateJobRunTriggers moves before DropEmptyStrings, which would drop a watched "" value and leave an entry that sets nothing. Co-authored-by: Isaac --- .../job_runs-on_value_change/databricks.yml | 2 +- .../job_runs-on_value_change/mutators.py | 5 - .../job_runs-on_value_change/output.txt | 54 ++++---- .../python/job_runs-on_value_change/script | 11 +- .../bundle/refschema/output/job_runs.txt | 4 +- .../job_runs/on_value_change/databricks.yml | 10 +- .../job_runs/on_value_change/output.txt | 90 ++++-------- .../resources/job_runs/on_value_change/script | 41 ++---- .../job_runs/on_value_change_empty/output.txt | 6 +- .../on_value_change_errors/databricks.yml | 2 +- .../on_value_change_errors/output.txt | 28 +--- .../job_runs/on_value_change_errors/script | 8 -- .../job_runs/on_value_change_ref/output.txt | 8 +- bundle/bundle.go | 5 - .../mutator/resolve_job_run_value_triggers.go | 46 ++++++ .../resolve_job_run_value_triggers_test.go | 27 ++++ .../process_static_resources.go | 3 - .../resourcemutator/resource_mutator.go | 3 - .../snapshot_job_run_value_triggers.go | 131 ------------------ .../snapshot_job_run_value_triggers_test.go | 49 ------- .../mutator/validate_job_run_triggers.go | 6 - bundle/config/resources/job_run.go | 2 +- bundle/direct/dresources/job_run.go | 44 ++++-- bundle/direct/dresources/job_run_test.go | 10 +- bundle/internal/schema/annotations.yml | 2 +- bundle/phases/initialize.go | 8 +- bundle/schema/jsonschema.json | 2 +- .../job_runs/_models/job_run_trigger.py | 4 +- 28 files changed, 198 insertions(+), 413 deletions(-) create mode 100644 bundle/config/mutator/resolve_job_run_value_triggers.go create mode 100644 bundle/config/mutator/resolve_job_run_value_triggers_test.go delete mode 100644 bundle/config/mutator/snapshot_job_run_value_triggers.go delete mode 100644 bundle/config/mutator/snapshot_job_run_value_triggers_test.go diff --git a/acceptance/bundle/python/job_runs-on_value_change/databricks.yml b/acceptance/bundle/python/job_runs-on_value_change/databricks.yml index acef9e1708a..dca8ab672c0 100644 --- a/acceptance/bundle/python/job_runs-on_value_change/databricks.yml +++ b/acceptance/bundle/python/job_runs-on_value_change/databricks.yml @@ -13,7 +13,7 @@ python: resources: - "resources:load_resources" mutators: - - "mutators:update_job_run" + - "mutators:watch_y" resources: jobs: diff --git a/acceptance/bundle/python/job_runs-on_value_change/mutators.py b/acceptance/bundle/python/job_runs-on_value_change/mutators.py index 91c20861407..15ef3dc2f58 100644 --- a/acceptance/bundle/python/job_runs-on_value_change/mutators.py +++ b/acceptance/bundle/python/job_runs-on_value_change/mutators.py @@ -4,11 +4,6 @@ from databricks.bundles.job_runs import JobRun, JobRunLifecycle, JobRunTrigger -@job_run_mutator -def update_job_run(run: JobRun) -> JobRun: - return replace(run, job_parameters={"source": "mutator"}) - - @job_run_mutator def watch_y(run: JobRun) -> JobRun: lifecycle = run.lifecycle or JobRunLifecycle() diff --git a/acceptance/bundle/python/job_runs-on_value_change/output.txt b/acceptance/bundle/python/job_runs-on_value_change/output.txt index deec2a3b5af..4374e23aa7f 100644 --- a/acceptance/bundle/python/job_runs-on_value_change/output.txt +++ b/acceptance/bundle/python/job_runs-on_value_change/output.txt @@ -1,40 +1,42 @@ -=== validate resolves a watch from YAML and from Python, with a mutator applied +=== validate resolves watches from YAML, from Python and from a Python mutator >>> uv run [UV_ARGS] -q [CLI] bundle validate --output json { "py_run": { "job_id": "${resources.jobs.my_job.id}", - "job_parameters": { - "source": "mutator" - }, "lifecycle": { "triggers": [ { "on_value_change": "x1" + }, + { + "on_value_change": "y1" } ], "triggers_state": { - "on_value_change": { - "${var.x}": "x1" - } + "on_value_change": [ + "x1", + "y1" + ] } } }, "yaml_run": { "job_id": "${resources.jobs.my_job.id}", - "job_parameters": { - "source": "mutator" - }, "lifecycle": { "triggers": [ { "on_value_change": "x1" + }, + { + "on_value_change": "y1" } ], "triggers_state": { - "on_value_change": { - "${var.x}": "x1" - } + "on_value_change": [ + "x1", + "y1" + ] } } } @@ -53,7 +55,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> uv run [UV_ARGS] -q [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged -=== changing the watched variable re-fires both runs +=== changing the variable watched in YAML and Python re-fires both runs >>> uv run [UV_ARGS] -q [CLI] bundle plan recreate job_runs.py_run recreate job_runs.yaml_run @@ -67,18 +69,16 @@ Recreated job_runs.yaml_run Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 2 deleted, 1 unchanged -=== a Python mutator cannot add a trigger to a job_run that sets on_value_change ->>> musterr uv run [UV_ARGS] -q [CLI] bundle validate -Error: Python mutators cannot add or remove lifecycle.triggers of a job_run that sets on_value_change - in resources.py:7:1 - -Error: Python mutators cannot add or remove lifecycle.triggers of a job_run that sets on_value_change - in databricks.yml:32:11 +=== changing the variable the Python mutator watches re-fires both runs +>>> uv run [UV_ARGS] -q [CLI] bundle plan +recreate job_runs.py_run +recreate job_runs.yaml_run -Name: test-bundle -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default +Plan: 2 to add, 0 to change, 2 to delete, 1 unchanged -Found 2 errors +>>> uv run [UV_ARGS] -q [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Recreated job_runs.py_run +Recreated job_runs.yaml_run +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 2 deleted, 1 unchanged diff --git a/acceptance/bundle/python/job_runs-on_value_change/script b/acceptance/bundle/python/job_runs-on_value_change/script index e7dbd54c474..bb4aa13a287 100644 --- a/acceptance/bundle/python/job_runs-on_value_change/script +++ b/acceptance/bundle/python/job_runs-on_value_change/script @@ -1,4 +1,4 @@ -title "validate resolves a watch from YAML and from Python, with a mutator applied" +title "validate resolves watches from YAML, from Python and from a Python mutator" trace uv run $UV_ARGS -q $CLI bundle validate --output json | jq .resources.job_runs title "first deploy runs both" @@ -7,13 +7,14 @@ trace uv run $UV_ARGS -q $CLI bundle deploy title "redeploy with unchanged values plans nothing" trace uv run $UV_ARGS -q $CLI bundle plan -title "changing the watched variable re-fires both runs" +title "changing the variable watched in YAML and Python re-fires both runs" update_file.py databricks.yml "default: x1" "default: x2" trace uv run $UV_ARGS -q $CLI bundle plan trace uv run $UV_ARGS -q $CLI bundle deploy -title "a Python mutator cannot add a trigger to a job_run that sets on_value_change" -update_file.py databricks.yml '"mutators:update_job_run"' '"mutators:watch_y"' -trace musterr uv run $UV_ARGS -q $CLI bundle validate +title "changing the variable the Python mutator watches re-fires both runs" +update_file.py databricks.yml "default: y1" "default: y2" +trace uv run $UV_ARGS -q $CLI bundle plan +trace uv run $UV_ARGS -q $CLI bundle deploy rm -fr .databricks __pycache__ diff --git a/acceptance/bundle/refschema/output/job_runs.txt b/acceptance/bundle/refschema/output/job_runs.txt index cfeca5028b9..8cca9edd2e0 100644 --- a/acceptance/bundle/refschema/output/job_runs.txt +++ b/acceptance/bundle/refschema/output/job_runs.txt @@ -24,8 +24,8 @@ resources.job_runs.*.lifecycle.triggers_state *resources.JobRunTriggersState INP resources.job_runs.*.lifecycle.triggers_state.on_bundle_deploy string INPUT STATE resources.job_runs.*.lifecycle.triggers_state.on_file_change map[string]string INPUT STATE resources.job_runs.*.lifecycle.triggers_state.on_file_change.* string INPUT STATE -resources.job_runs.*.lifecycle.triggers_state.on_value_change map[string]string INPUT STATE -resources.job_runs.*.lifecycle.triggers_state.on_value_change.* string INPUT STATE +resources.job_runs.*.lifecycle.triggers_state.on_value_change []string INPUT STATE +resources.job_runs.*.lifecycle.triggers_state.on_value_change[*] string INPUT STATE resources.job_runs.*.modified_status string INPUT resources.job_runs.*.notebook_params map[string]string ALL resources.job_runs.*.notebook_params.* string ALL diff --git a/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml b/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml index cb2511f5c00..d46e21e06ef 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml @@ -8,8 +8,6 @@ variables: default: a1 number: default: 42 - long: - default: a-long-non-hex-watched-value-used-to-exercise-a-lengthy-string-zzz resources: jobs: @@ -24,10 +22,4 @@ resources: my_run: job_id: ${resources.jobs.my_job.id} lifecycle: - triggers: - - on_file_change: watched.txt - - on_value_change: " ${var.stamp} " - - on_value_change: ${var.other} - - on_value_change: ${var.number} - - on_value_change: ${workspace.root_path} - - on_value_change: ${var.long} + triggers: [{on_file_change: watched.txt}, {on_value_change: "${var.stamp}"}, {on_value_change: "${var.other}"}, {on_value_change: "${var.number}"}, {on_value_change: "${workspace.root_path}"}] diff --git a/acceptance/bundle/resources/job_runs/on_value_change/output.txt b/acceptance/bundle/resources/job_runs/on_value_change/output.txt index 63461bbb830..510452d70c3 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change/output.txt @@ -22,19 +22,18 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== resolved values are stored in state +=== resolved values are stored in state in trigger order >>> print_state.py { "on_file_change": { "watched.txt": "[FILE_HASH][0]" }, - "on_value_change": { - "${var.long}": "a-long-non-hex-watched-value-used-to-exercise-a-lengthy-string-zzz", - "${var.number}": "42", - "${var.other}": "a1", - "${var.stamp}": "v1", - "${workspace.root_path}": "/Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default" - } + "on_value_change": [ + "v1", + "a1", + "42", + "/Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default" + ] } === redeploy with unchanged values plans nothing @@ -49,14 +48,9 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now === changing one value re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - >>> changes { - "lifecycle.triggers_state.on_value_change['${var.stamp}']": { + "lifecycle.triggers_state.on_value_change[0]": { "action": "recreate", "reason": "immutable", "old": "v1", @@ -90,7 +84,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } -=== removing one on_value_change through a saved plan does not run +=== removing the last on_value_change through a saved plan does not run >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -101,30 +95,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now -=== re-adding a removed on_value_change with an unchanged value does not re-fire ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - ->>> changes -{ - "result_state": { - "action": "skip", - "reason": "remote_already_set", - "new": "SUCCESS", - "remote": "SUCCESS" - } -} - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... -Files: 1 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 2 unchanged - ->>> print_requests.py --keep //jobs/runs/delete - ->>> print_requests.py //jobs/run-now - -=== removing the same on_value_change again does not run +=== re-adding it with an unchanged value does not re-fire >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -135,19 +106,23 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now -=== changing a remaining watch to a removed value re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - +=== removing an on_value_change that is not the last shifts the rest and re-fires >>> changes { - "lifecycle.triggers_state.on_value_change['${var.stamp}']": { + "lifecycle.triggers_state.on_value_change": { "action": "recreate", "reason": "immutable", - "old": "v2", - "new": "a1" + "old": [ + "v2", + "a1", + "42", + "/Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default" + ], + "new": [ + "v2", + "42", + "/Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default" + ] }, "result_state": { "action": "skip", @@ -159,14 +134,12 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[WORKSPACE_ID] -Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[WORKSPACE_ID] +Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS Recreated job_runs.my_run Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged ->>> print_requests.py --keep //jobs/runs/delete - >>> print_requests.py //jobs/run-now { "method": "POST", @@ -177,7 +150,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } -=== removing the remaining on_value_change entries is unchanged and does not run +=== removing the remaining on_value_change entries does not run >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -189,11 +162,6 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now === file changes still re-fire after value triggers are removed ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - >>> changes { "lifecycle.triggers_state.on_file_change['watched.txt']": { @@ -212,14 +180,12 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_5]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_5]?o=[WORKSPACE_ID] -Output from job_runs.my_run: id=[MY_RUN_ID_5]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[WORKSPACE_ID] +Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS Recreated job_runs.my_run Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged ->>> print_requests.py --keep //jobs/runs/delete - >>> print_requests.py //jobs/run-now { "method": "POST", diff --git a/acceptance/bundle/resources/job_runs/on_value_change/script b/acceptance/bundle/resources/job_runs/on_value_change/script index 3aa2fe4b840..2075827b559 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/script +++ b/acceptance/bundle/resources/job_runs/on_value_change/script @@ -14,7 +14,7 @@ trace read_id.py my_job read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -title "resolved values are stored in state" +title "resolved values are stored in state in trigger order" trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle.triggers_state' title "redeploy with unchanged values plans nothing" @@ -24,64 +24,41 @@ trace print_requests.py //jobs/run-now title "changing one value re-fires" update_file.py databricks.yml "default: v1" "default: v2" -trace $CLI bundle plan trace changes trace $CLI bundle deploy read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now -title "removing one on_value_change through a saved plan does not run" -update_file.py databricks.yml ' - on_value_change: ${var.other} -' '' +title "removing the last on_value_change through a saved plan does not run" +update_file.py databricks.yml ', {on_value_change: "${workspace.root_path}"}' '' trace $CLI bundle plan $CLI bundle plan -o json > tmp.plan.json trace $CLI bundle deploy --plan tmp.plan.json trace print_requests.py //jobs/run-now -title "re-adding a removed on_value_change with an unchanged value does not re-fire" -update_file.py databricks.yml ' - on_value_change: " ${var.stamp} " -' ' - on_value_change: " ${var.stamp} " - - on_value_change: ${var.other} -' -trace $CLI bundle plan -trace changes -trace $CLI bundle deploy -read_id.py my_run > /dev/null -trace print_requests.py --keep //jobs/runs/delete -trace print_requests.py //jobs/run-now - -title "removing the same on_value_change again does not run" -update_file.py databricks.yml ' - on_value_change: ${var.other} -' '' +title "re-adding it with an unchanged value does not re-fire" +update_file.py databricks.yml '{on_value_change: "${var.number}"}' '{on_value_change: "${var.number}"}, {on_value_change: "${workspace.root_path}"}' trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //jobs/run-now -title "changing a remaining watch to a removed value re-fires" -update_file.py databricks.yml "default: v2" "default: a1" -trace $CLI bundle plan +title "removing an on_value_change that is not the last shifts the rest and re-fires" +update_file.py databricks.yml ', {on_value_change: "${var.other}"}' '' trace changes trace $CLI bundle deploy read_id.py my_run > /dev/null -trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now -title "removing the remaining on_value_change entries is unchanged and does not run" -update_file.py databricks.yml ' - on_value_change: " ${var.stamp} " - - on_value_change: ${var.number} - - on_value_change: ${workspace.root_path} - - on_value_change: ${var.long} -' '' +title "removing the remaining on_value_change entries does not run" +update_file.py databricks.yml ', {on_value_change: "${var.stamp}"}, {on_value_change: "${var.number}"}, {on_value_change: "${workspace.root_path}"}' '' trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //jobs/run-now title "file changes still re-fire after value triggers are removed" update_file.py watched.txt "v1" "v2" -trace $CLI bundle plan trace changes trace $CLI bundle deploy read_id.py my_run > /dev/null -trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt index 8e2ef7cbae1..abe2513fd7d 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt @@ -22,9 +22,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_state.py { "triggers_state": { - "on_value_change": { - "${var.optional}": "" - } + "on_value_change": [ + "" + ] } } diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_value_change_errors/databricks.yml index b8eb272e06a..43ab45ad51e 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_errors/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/databricks.yml @@ -21,4 +21,4 @@ resources: my_run: job_id: ${resources.jobs.my_job.id} lifecycle: - triggers: [{on_value_change: " "}] + triggers: [{on_file_change: watched.txt, on_value_change: "${var.value}"}] diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt index 46ada1c6590..eaf3d16b41c 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/output.txt @@ -1,30 +1,4 @@ -=== an empty expression is rejected ->>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_value_change must be non-empty when set - in databricks.yml:24:38 - -Name: job-runs-on-value-change-errors -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-errors/default - -Found 1 error - -=== duplicate expressions are rejected after trimming ->>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_value_change expressions must be unique - in databricks.yml:24:73 - -Name: job-runs-on-value-change-errors -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-errors/default - -Found 1 error - === multiple trigger fields in one entry are rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers entry must set only one of on_bundle_deploy, on_file_change, or on_value_change @@ -56,7 +30,7 @@ Warning: expected string, found map at resources.job_runs.my_run.lifecycle.triggers[0].on_value_change in databricks.yml:24:38 -Error: lifecycle.triggers.on_value_change must resolve to a string, number or boolean +Error: lifecycle.triggers entry must set on_bundle_deploy, on_file_change, or on_value_change in databricks.yml:24:20 Name: job-runs-on-value-change-errors diff --git a/acceptance/bundle/resources/job_runs/on_value_change_errors/script b/acceptance/bundle/resources/job_runs/on_value_change_errors/script index 90d776d498c..f9b543f8c1e 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_value_change_errors/script @@ -1,12 +1,4 @@ -title "an empty expression is rejected" -trace musterr $CLI bundle validate - -title "duplicate expressions are rejected after trimming" -update_file.py databricks.yml ' triggers: [{on_value_change: " "}]' ' triggers: [{on_value_change: "${var.value}"}, {on_value_change: " ${var.value} "}]' -trace musterr $CLI bundle validate - title "multiple trigger fields in one entry are rejected" -update_file.py databricks.yml ' triggers: [{on_value_change: "${var.value}"}, {on_value_change: " ${var.value} "}]' ' triggers: [{on_file_change: watched.txt, on_value_change: "${var.value}"}]' trace musterr $CLI bundle validate title "an undefined variable is rejected" diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt index 4dec3f6d6b1..727b5c36756 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt @@ -49,10 +49,10 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged === state records resolved values for pure and mixed references >>> print_state.py -{ - "${resources.jobs.other.id}": "[OTHER_ID]", - "${resources.jobs.other.id}-${var.suffix}-${resources.jobs.extra.id}": "[OTHER_ID]-suffix-[EXTRA_ID]" -} +[ + "[OTHER_ID]", + "[OTHER_ID]-suffix-[EXTRA_ID]" +] === redeploy with unchanged ids plans nothing >>> [CLI] bundle plan diff --git a/bundle/bundle.go b/bundle/bundle.go index 91b1c5e4e57..6762b38b91e 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -161,11 +161,6 @@ type Bundle struct { // to end in those two segments does not count. RootPathIsNameTargetScoped bool - // JobRunValueTriggers holds, per job_run, the on_value_change expression of each - // lifecycle.triggers entry ("" for other entries), recorded before variable - // resolution. ResolveJobRunValueTriggers keys the resolved values by them. - JobRunValueTriggers map[string][]string - // Metadata about the bundle deployment. This is the interface Databricks services // rely on to integrate with bundles when they need additional information about // a bundle deployment. diff --git a/bundle/config/mutator/resolve_job_run_value_triggers.go b/bundle/config/mutator/resolve_job_run_value_triggers.go new file mode 100644 index 00000000000..b3539aacf76 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_value_triggers.go @@ -0,0 +1,46 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" +) + +type resolveJobRunValueTriggers struct{} + +// ResolveJobRunValueTriggers copies each job_run's interpolated on_value_change values, +// in trigger order, into lifecycle.triggers_state, where the planner compares them by +// position with the values recorded when the run last fired. +func ResolveJobRunValueTriggers() bundle.Mutator { + return &resolveJobRunValueTriggers{} +} + +func (*resolveJobRunValueTriggers) Name() string { + return "ResolveJobRunValueTriggers" +} + +func (*resolveJobRunValueTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, jr := range b.Config.Resources.JobRuns { + if jr == nil || jr.Lifecycle == nil { + continue + } + + var values []string + for _, trigger := range jr.Lifecycle.Triggers { + if trigger.OnValueChange != nil { + values = append(values, *trigger.OnValueChange) + } + } + if values == nil { + continue + } + + if jr.Lifecycle.TriggersState == nil { + jr.Lifecycle.TriggersState = &resources.JobRunTriggersState{} + } + jr.Lifecycle.TriggersState.OnValueChange = values + } + return nil +} diff --git a/bundle/config/mutator/resolve_job_run_value_triggers_test.go b/bundle/config/mutator/resolve_job_run_value_triggers_test.go new file mode 100644 index 00000000000..8968fb543fb --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_value_triggers_test.go @@ -0,0 +1,27 @@ +package mutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" +) + +func TestResolveJobRunValueTriggers(t *testing.T) { + enabled := true + first := "first" + empty := "" + b := &bundle.Bundle{Config: config.Root{ + Resources: config.Resources{JobRuns: map[string]*resources.JobRun{"run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{{OnValueChange: &first}, {OnBundleDeploy: &enabled}, {OnValueChange: &empty}}, + }, + }}}, + }} + + assert.Empty(t, bundle.Apply(t.Context(), b, mutator.ResolveJobRunValueTriggers())) + assert.Equal(t, []string{"first", ""}, b.Config.Resources.JobRuns["run"].Lifecycle.TriggersState.OnValueChange) +} diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index 920bbbfc6f6..06ec593f092 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -41,9 +41,6 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia bundle.ApplySeqContext( ctx, b, - // Record on_value_change expressions before variable resolution replaces them. - mutator.SnapshotJobRunValueTriggers(), - // Reads (dynamic): * (strings) (searches for variable references in string values) // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 82c5422bc80..e00444cc773 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -139,9 +139,6 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { validate.SingleNodeCluster(), - // Record on_value_change expressions before variable resolution replaces them. - mutator.SnapshotJobRunValueTriggers(), - // Reads (dynamic): * (strings) (searches for variable references in string values) // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes diff --git a/bundle/config/mutator/snapshot_job_run_value_triggers.go b/bundle/config/mutator/snapshot_job_run_value_triggers.go deleted file mode 100644 index f53f1a7a2bb..00000000000 --- a/bundle/config/mutator/snapshot_job_run_value_triggers.go +++ /dev/null @@ -1,131 +0,0 @@ -package mutator - -import ( - "context" - "fmt" - "maps" - "slices" - "strings" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/diag" -) - -type snapshotJobRunValueTriggers struct{} - -// SnapshotJobRunValueTriggers records each job_run's on_value_change expressions in -// b.JobRunValueTriggers before variable resolution, so that ResolveJobRunValueTriggers -// can key the resolved values by the expression. A job_run that is already recorded is -// skipped: when a Python mutator updates it, its triggers are already resolved. -func SnapshotJobRunValueTriggers() bundle.Mutator { - return &snapshotJobRunValueTriggers{} -} - -func (*snapshotJobRunValueTriggers) Name() string { - return "SnapshotJobRunValueTriggers" -} - -func (*snapshotJobRunValueTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - // Sorted so diagnostics from several job_runs come out in a stable order. - for _, name := range slices.Sorted(maps.Keys(b.Config.Resources.JobRuns)) { - jr := b.Config.Resources.JobRuns[name] - if jr == nil || jr.Lifecycle == nil || b.JobRunValueTriggers[name] != nil { - continue - } - - exprs := make([]string, len(jr.Lifecycle.Triggers)) - seen := make(map[string]bool) - for i, trigger := range jr.Lifecycle.Triggers { - if trigger.OnValueChange == nil { - continue - } - - path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_value_change", name, i) - expr := strings.TrimSpace(*trigger.OnValueChange) - if expr == "" { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_value_change must be non-empty when set", - Locations: b.Config.GetLocations(path), - }) - continue - } - if seen[expr] { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_value_change expressions must be unique", - Locations: b.Config.GetLocations(path), - }) - continue - } - seen[expr] = true - exprs[i] = expr - } - - if len(seen) > 0 { - if b.JobRunValueTriggers == nil { - b.JobRunValueTriggers = make(map[string][]string) - } - b.JobRunValueTriggers[name] = exprs - } - } - return diags -} - -type resolveJobRunValueTriggers struct{} - -// ResolveJobRunValueTriggers stores each job_run's resolved on_value_change values in -// lifecycle.triggers_state, keyed by the expressions SnapshotJobRunValueTriggers -// recorded. The values are paired with the expressions by trigger position, so a -// Python mutator must not add or remove triggers of a job_run that sets on_value_change. -func ResolveJobRunValueTriggers() bundle.Mutator { - return &resolveJobRunValueTriggers{} -} - -func (*resolveJobRunValueTriggers) Name() string { - return "ResolveJobRunValueTriggers" -} - -func (*resolveJobRunValueTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - for _, name := range slices.Sorted(maps.Keys(b.JobRunValueTriggers)) { - exprs := b.JobRunValueTriggers[name] - jr := b.Config.Resources.JobRuns[name] - path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) - if jr.Lifecycle == nil || len(jr.Lifecycle.Triggers) != len(exprs) { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "Python mutators cannot add or remove lifecycle.triggers of a job_run that sets on_value_change", - Locations: b.Config.GetLocations(path), - }) - continue - } - - values := make(map[string]string) - for i, expr := range exprs { - if expr == "" { - continue - } - value := jr.Lifecycle.Triggers[i].OnValueChange - if value == nil { - // Normalization dropped a value that is a map or a list, with a warning. - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_value_change must resolve to a string, number or boolean", - Locations: b.Config.GetLocations(fmt.Sprintf("%s[%d]", path, i)), - }) - continue - } - // Trimmed like the expression, so padding the expression does not re-fire. - values[expr] = strings.TrimSpace(*value) - } - - if jr.Lifecycle.TriggersState == nil { - jr.Lifecycle.TriggersState = &resources.JobRunTriggersState{} - } - jr.Lifecycle.TriggersState.OnValueChange = values - } - return diags -} diff --git a/bundle/config/mutator/snapshot_job_run_value_triggers_test.go b/bundle/config/mutator/snapshot_job_run_value_triggers_test.go deleted file mode 100644 index 99e597a94eb..00000000000 --- a/bundle/config/mutator/snapshot_job_run_value_triggers_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package mutator_test - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/config/resources" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestSnapshotJobRunValueTriggers(t *testing.T) { - enabled := true - padded := " ${var.value} " - duplicate := "${var.value}" - b := &bundle.Bundle{Config: config.Root{ - Resources: config.Resources{JobRuns: map[string]*resources.JobRun{"run": { - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &enabled}, {OnValueChange: &padded}, {OnValueChange: &duplicate}}, - }, - }}}, - }} - - diags := bundle.Apply(t.Context(), b, mutator.SnapshotJobRunValueTriggers()) - require.Len(t, diags, 1) - assert.Equal(t, "lifecycle.triggers.on_value_change expressions must be unique", diags[0].Summary) - assert.Equal(t, map[string][]string{"run": {"", "${var.value}", ""}}, b.JobRunValueTriggers) -} - -func TestResolveJobRunValueTriggers(t *testing.T) { - resolved := " resolved " - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{JobRuns: map[string]*resources.JobRun{"run": { - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{{OnValueChange: &resolved}, {}}, - }, - }}}, - }, - JobRunValueTriggers: map[string][]string{"run": {"${var.value}", "${var.complex}"}}, - } - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunValueTriggers()) - require.Len(t, diags, 1) - assert.Equal(t, "lifecycle.triggers.on_value_change must resolve to a string, number or boolean", diags[0].Summary) - assert.Equal(t, map[string]string{"${var.value}": "resolved"}, b.Config.Resources.JobRuns["run"].Lifecycle.TriggersState.OnValueChange) -} diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index 106352fe596..cb12587a306 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -26,15 +26,9 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D if jr == nil || jr.Lifecycle == nil { continue } - recorded := b.JobRunValueTriggers[name] for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) armed := t.ArmedCount() - // DropEmptyStrings removes an on_value_change whose value is "", which is a - // valid value to watch; the expression is still recorded. - if t.OnValueChange == nil && i < len(recorded) && recorded[i] != "" { - armed++ - } if armed == 0 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 5b681f7fe6b..147155994a8 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -67,7 +67,7 @@ func (t JobRunTrigger) ArmedCount() int { type JobRunTriggersState struct { OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` OnFileChange map[string]string `json:"on_file_change,omitempty"` - OnValueChange map[string]string `json:"on_value_change,omitempty"` + OnValueChange []string `json:"on_value_change,omitempty"` // in lifecycle.triggers order } // IsEmpty reports whether no trigger is armed. An empty state is left off the diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index d122b688839..87fd1a68adf 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -367,8 +367,10 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // stopped without succeeding keeps its recreate. A SKIPPED run reports no // result_state either, so the lifecycle state is what tells the two apart. // Removing a trigger drops the change and leaves the last fingerprint in state, -// so re-adding the trigger only re-fires when the watched files or value -// changed meanwhile. All other trigger changes recreate the run. +// so re-adding the trigger only re-fires when the watched files changed +// meanwhile. on_value_change values are compared by position, so removing the +// last watches drops the change, and removing or reordering any other watch +// re-fires. All other trigger changes recreate the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { switch path.String() { case "lifecycle": @@ -382,10 +384,17 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa if change.New == nil || change.New == "" { change.Reason = deployplan.ReasonDrop } - case "lifecycle.triggers_state.on_file_change", "lifecycle.triggers_state.on_value_change": - // An emptied map is a removal; a map with both sides present has its - // per-key entries (below) classify the change. - if isEmptyTriggerMap(change.New) || change.Old != nil { + case "lifecycle.triggers_state.on_file_change": + // As above: an emptied map is a removal, and pattern entries classify a map + // that still has both sides. + if isEmptyFileTriggerMap(change.New) || change.Old != nil { + change.Reason = deployplan.ReasonDrop + } + case "lifecycle.triggers_state.on_value_change": + // Lists of equal length are diffed per position, and any change there + // re-fires. A shorter list that keeps the leading values means watches + // were removed from the end. + if isLeadingValues(change.Old, change.New) { change.Reason = deployplan.ReasonDrop } case "result_state": @@ -396,15 +405,12 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Action = deployplan.Skip change.Reason = "run in progress" default: - // A single on_file_change pattern or on_value_change expression entry, e.g. - // lifecycle.triggers_state.on_file_change['seed.txt']. Removing one entry + // A single on_file_change pattern entry, e.g. + // lifecycle.triggers_state.on_file_change['seed.txt']. Removing one pattern // drops its change and leaves the last fingerprint in state. - if parent := path.Parent(); parent != nil { - switch parent.String() { - case "lifecycle.triggers_state.on_file_change", "lifecycle.triggers_state.on_value_change": - if change.New == nil { - change.Reason = deployplan.ReasonDrop - } + if parent := path.Parent(); parent != nil && parent.String() == "lifecycle.triggers_state.on_file_change" { + if change.New == nil { + change.Reason = deployplan.ReasonDrop } } } @@ -416,7 +422,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa // re-fires) leaves the run alone on the backend and only drops the state entry // (see PlanEntry.StateOnly). A recreate still triggers a fresh run. -func isEmptyTriggerMap(v any) bool { +func isEmptyFileTriggerMap(v any) bool { if v == nil { return true } @@ -424,6 +430,14 @@ func isEmptyTriggerMap(v any) bool { return ok && len(m) == 0 } +// isLeadingValues reports whether newV holds the leading values of oldV and fewer +// of them (none, when newV is nil). +func isLeadingValues(oldV, newV any) bool { + oldValues, _ := oldV.([]string) + newValues, _ := newV.([]string) + return len(newValues) < len(oldValues) && slices.Equal(oldValues[:len(newValues)], newValues) +} + func parseRunID(id string) (int64, error) { result, err := strconv.ParseInt(id, 10, 64) if err != nil { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 3b6735549b8..b3d97f5e2d6 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -271,10 +271,8 @@ func TestJobRunStateOmitsEmptyLifecycle(t *testing.T) { func TestJobRunPrepareStateCopiesResolvedTriggers(t *testing.T) { enabled := true triggers := &resources.JobRunTriggersState{ - OnFileChange: map[string]string{"*.txt": "hash"}, - OnValueChange: map[string]string{ - "${var.watched}": "resolved", - }, + OnFileChange: map[string]string{"*.txt": "hash"}, + OnValueChange: []string{"resolved"}, } input := &resources.JobRun{ Lifecycle: &resources.JobRunLifecycle{ @@ -289,9 +287,7 @@ func TestJobRunPrepareStateCopiesResolvedTriggers(t *testing.T) { require.NotNil(t, state.Lifecycle.TriggersState) assert.NotSame(t, triggers, state.Lifecycle.TriggersState) assert.Equal(t, triggers.OnFileChange, state.Lifecycle.TriggersState.OnFileChange) - assert.Equal(t, map[string]string{ - "${var.watched}": "resolved", - }, state.Lifecycle.TriggersState.OnValueChange) + assert.Equal(t, triggers.OnValueChange, state.Lifecycle.TriggersState.OnValueChange) assert.NotEmpty(t, state.Lifecycle.TriggersState.OnBundleDeploy) assert.Empty(t, triggers.OnBundleDeploy) } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 1a2a72d114f..e982a623a0a 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1135,7 +1135,7 @@ resources: Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. "on_value_change": "description": |- - Interpolated expression. Re-fire the run when its resolved value differs from the value recorded when the run last fired, or when no value was recorded for it. Resource references in the expression become deployment dependencies. Removing a watch does not re-fire the run. + Interpolated expression. Re-fire the run when its resolved value changes. Values are compared by position among the on_value_change entries: adding one re-fires the run, removing the last ones does not, and removing or reordering any other one does. Resource references in the expression become deployment dependencies. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 823408061db..d870b5e872d 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -171,6 +171,11 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // model, so exposing it to Python would fail resource loading. mutator.ResolveJobRunValueTriggers(), + // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). + // Runs before DropEmptyStrings, which would drop an on_value_change whose value + // is "" (a valid value to watch) and leave an entry that sets nothing. + mutator.ValidateJobRunTriggers(), + // Compute resources.volumes.*.volume_path and resolve references to it. Must run after // PythonMutator: volume_path is computed and read-only, not declared by the PyDABs Volume // model, so exposing it to Python would fail resource loading (like "deployment" below). @@ -219,9 +224,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Rejects a Data API that sets nothing besides parent; the API would reject it at deploy time. validate.PostgresDataApiSettings(), - // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). - mutator.ValidateJobRunTriggers(), - // Reads (dynamic): resources.*.*.lifecycle.depends_on // Rejects entries that point to a missing or ambiguous resource, or to the resource itself. validate.LifecycleDependsOn(), diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index a9b89268b9d..972247fea8a 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1710,7 +1710,7 @@ "$ref": "#/$defs/string" }, "on_value_change": { - "description": "Interpolated expression. Re-fire the run when its resolved value differs from the value recorded when the run last fired, or when no value was recorded for it. Resource references in the expression become deployment dependencies. Removing a watch does not re-fire the run.", + "description": "Interpolated expression. Re-fire the run when its resolved value changes. Values are compared by position among the on_value_change entries: adding one re-fires the run, removing the last ones does not, and removing or reordering any other one does. Resource references in the expression become deployment dependencies.", "$ref": "#/$defs/string" } }, diff --git a/python/databricks/bundles/job_runs/_models/job_run_trigger.py b/python/databricks/bundles/job_runs/_models/job_run_trigger.py index a25a0e39959..5180dd9947d 100644 --- a/python/databricks/bundles/job_runs/_models/job_run_trigger.py +++ b/python/databricks/bundles/job_runs/_models/job_run_trigger.py @@ -27,7 +27,7 @@ class JobRunTrigger: on_value_change: VariableOrOptional[str] = None """ - Interpolated expression. Re-fire the run when its resolved value differs from the value recorded when the run last fired, or when no value was recorded for it. Resource references in the expression become deployment dependencies. Removing a watch does not re-fire the run. + Interpolated expression. Re-fire the run when its resolved value changes. Values are compared by position among the on_value_change entries: adding one re-fires the run, removing the last ones does not, and removing or reordering any other one does. Resource references in the expression become deployment dependencies. """ @classmethod @@ -53,7 +53,7 @@ class JobRunTriggerDict(TypedDict, total=False): on_value_change: VariableOrOptional[str] """ - Interpolated expression. Re-fire the run when its resolved value differs from the value recorded when the run last fired, or when no value was recorded for it. Resource references in the expression become deployment dependencies. Removing a watch does not re-fire the run. + Interpolated expression. Re-fire the run when its resolved value changes. Values are compared by position among the on_value_change entries: adding one re-fires the run, removing the last ones does not, and removing or reordering any other one does. Resource references in the expression become deployment dependencies. """ From 5d9dfa5e4beec14bc8b5b8876238e740a64cdfee Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 16:44:26 +0100 Subject: [PATCH 12/12] Keep empty strings in lifecycle in DropEmptyStrings DropEmptyStrings exists so "" on an omitempty field is not force-sent to the backend. lifecycle is never sent, and there an empty string is a value: job_runs lifecycle.triggers.on_value_change can watch a variable that is "". Dropping it left a trigger entry that sets nothing. With lifecycle left alone, ValidateJobRunTriggers goes back to its place after DropEmptyStrings. Co-authored-by: Isaac --- .../job_runs/on_value_change_empty/output.txt | 15 +++++++++++ .../job_runs/on_value_change_empty/script | 3 +++ bundle/config/mutator/drop_empty_strings.go | 25 ++++++++++++++++++- bundle/phases/initialize.go | 8 +++--- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt index abe2513fd7d..48701895017 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_empty/output.txt @@ -1,4 +1,19 @@ +=== validate keeps the empty value +>>> [CLI] bundle validate -o json +{ + "triggers": [ + { + "on_value_change": "" + } + ], + "triggers_state": { + "on_value_change": [ + "" + ] + } +} + === a watch on an empty value deploys and records it >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... diff --git a/acceptance/bundle/resources/job_runs/on_value_change_empty/script b/acceptance/bundle/resources/job_runs/on_value_change_empty/script index d0e9aca93bd..136dc0aaa21 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_empty/script +++ b/acceptance/bundle/resources/job_runs/on_value_change_empty/script @@ -4,6 +4,9 @@ cleanup() { } trap cleanup EXIT +title "validate keeps the empty value" +trace $CLI bundle validate -o json | jq .resources.job_runs.my_run.lifecycle + title "a watch on an empty value deploys and records it" trace $CLI bundle deploy read_id.py my_job > /dev/null diff --git a/bundle/config/mutator/drop_empty_strings.go b/bundle/config/mutator/drop_empty_strings.go index 028351ed943..83794505dd8 100644 --- a/bundle/config/mutator/drop_empty_strings.go +++ b/bundle/config/mutator/drop_empty_strings.go @@ -12,6 +12,9 @@ import ( type dropEmptyStrings struct{} +// lifecyclePattern matches the lifecycle block of every resource. +var lifecyclePattern = dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("lifecycle")) + // DropEmptyStrings removes empty-string values on omitempty resource fields, so // they are not force-sent to the backend. An empty string reaches this point // either literally (policy_id: "") or via a variable that resolved to "", so @@ -21,6 +24,10 @@ type dropEmptyStrings struct{} // force-sends explicitly-set zero values, defeating the omitempty tag. Dropping // here fixes it and makes the result visible in `bundle validate -o json`, which // serializes the dynamic value. +// +// lifecycle is left as is: it configures the bundle and is never sent to the +// backend, and an empty string is meaningful there (job_runs +// lifecycle.triggers.on_value_change can watch a value that is ""). func DropEmptyStrings() bundle.Mutator { return &dropEmptyStrings{} } @@ -32,7 +39,16 @@ func (m *dropEmptyStrings) Name() string { func (m *dropEmptyStrings) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - root, err := dyn.Map(root, "resources", func(_ dyn.Path, resources dyn.Value) (dyn.Value, error) { + lifecycles := make(map[string]dyn.Value) + _, err := dyn.MapByPattern(root, lifecyclePattern, func(p dyn.Path, lifecycle dyn.Value) (dyn.Value, error) { + lifecycles[p.String()] = lifecycle + return lifecycle, nil + }) + if err != nil { + return root, err + } + + root, err = dyn.Map(root, "resources", func(_ dyn.Path, resources dyn.Value) (dyn.Value, error) { // Normalize against the resources type so omitempty is known per field. // Only DropEmptyStrings is set: existing values are kept as-is otherwise. out, normDiags := convert.Normalize(config.Resources{}, resources, convert.DropEmptyStrings) @@ -43,6 +59,13 @@ func (m *dropEmptyStrings) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia return root, err } + root, err = dyn.MapByPattern(root, lifecyclePattern, func(p dyn.Path, _ dyn.Value) (dyn.Value, error) { + return lifecycles[p.String()], nil + }) + if err != nil { + return root, err + } + // Keep an empty apps description instead of dropping it: an app update without // the field leaves the remote description unchanged, so removing description // from config would never converge. Force-sending "" clears it. diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index d870b5e872d..823408061db 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -171,11 +171,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // model, so exposing it to Python would fail resource loading. mutator.ResolveJobRunValueTriggers(), - // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). - // Runs before DropEmptyStrings, which would drop an on_value_change whose value - // is "" (a valid value to watch) and leave an entry that sets nothing. - mutator.ValidateJobRunTriggers(), - // Compute resources.volumes.*.volume_path and resolve references to it. Must run after // PythonMutator: volume_path is computed and read-only, not declared by the PyDABs Volume // model, so exposing it to Python would fail resource loading (like "deployment" below). @@ -224,6 +219,9 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Rejects a Data API that sets nothing besides parent; the API would reject it at deploy time. validate.PostgresDataApiSettings(), + // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). + mutator.ValidateJobRunTriggers(), + // Reads (dynamic): resources.*.*.lifecycle.depends_on // Rejects entries that point to a missing or ambiguous resource, or to the resource itself. validate.LifecycleDependsOn(),