Add job_runs lifecycle.triggers.on_value_change

.nextchanges/bundles/job-runs-on-value-change.md

@@ -0,0 +1 @@
+* Add `lifecycle.triggers.on_value_change` for `job_runs`: re-fire the run when a watched expression's resolved value changes. ([#6341](https://github.com/databricks/cli/pull/6341))

acceptance/bundle/python/job_runs-on_value_change/databricks.yml

@@ -0,0 +1,32 @@
+bundle:
+  name: test-bundle
+
+sync: {paths: []} # don't need to copy files
+
+variables:
+  x:
+    default: x1
+  y:
+    default: y1
+
+python:
+  resources:
+    - "resources:load_resources"
+  mutators:
+    - "mutators:watch_y"
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+  job_runs:
+    yaml_run:
+      job_id: ${resources.jobs.my_job.id}
+      lifecycle:
+        triggers:
+          - on_value_change: ${var.x}

acceptance/bundle/python/job_runs-on_value_change/mutators.py

@@ -0,0 +1,12 @@
+from dataclasses import replace
+
+from databricks.bundles.core import job_run_mutator
+from databricks.bundles.job_runs import JobRun, JobRunLifecycle, JobRunTrigger
+
+
+@job_run_mutator
+def watch_y(run: JobRun) -> JobRun:
+    lifecycle = run.lifecycle or JobRunLifecycle()
+    triggers = [*lifecycle.triggers, JobRunTrigger(on_value_change="${var.y}")]
+
+    return replace(run, lifecycle=replace(lifecycle, triggers=triggers))

acceptance/bundle/python/job_runs-on_value_change/out.test.toml

@@ -0,0 +1,3 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]
+EnvMatrix.PYDAB_VERSION = ["current"]

acceptance/bundle/python/job_runs-on_value_change/output.txt

@@ -0,0 +1,84 @@
+
+=== validate resolves watches from YAML, from Python and from a Python mutator
+>>> uv run [UV_ARGS] -q [CLI] bundle validate --output json
+{
+  "py_run": {
+    "job_id": "${resources.jobs.my_job.id}",
+    "lifecycle": {
+      "triggers": [
+        {
+          "on_value_change": "x1"
+        },
+        {
+          "on_value_change": "y1"
+        }
+      ],
+      "triggers_state": {
+        "on_value_change": [
+          "x1",
+          "y1"
+        ]
+      }
+    }
+  },
+  "yaml_run": {
+    "job_id": "${resources.jobs.my_job.id}",
+    "lifecycle": {
+      "triggers": [
+        {
+          "on_value_change": "x1"
+        },
+        {
+          "on_value_change": "y1"
+        }
+      ],
+      "triggers_state": {
+        "on_value_change": [
+          "x1",
+          "y1"
+        ]
+      }
+    }
+  }
+}
+
+=== first deploy runs both
+>>> uv run [UV_ARGS] -q [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Created job_runs.py_run
+Created job_runs.yaml_run
+Created jobs.my_job
+Files: 0 uploaded, 0 deleted
+Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
+
+=== redeploy with unchanged values plans nothing
+>>> uv run [UV_ARGS] -q [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged
+
+=== changing the variable watched in YAML and Python re-fires both runs
+>>> uv run [UV_ARGS] -q [CLI] bundle plan
+recreate job_runs.py_run
+recreate job_runs.yaml_run
+
+Plan: 2 to add, 0 to change, 2 to delete, 1 unchanged
+
+>>> uv run [UV_ARGS] -q [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Recreated job_runs.py_run
+Recreated job_runs.yaml_run
+Files: 0 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 2 deleted, 1 unchanged
+
+=== changing the variable the Python mutator watches re-fires both runs
+>>> uv run [UV_ARGS] -q [CLI] bundle plan
+recreate job_runs.py_run
+recreate job_runs.yaml_run
+
+Plan: 2 to add, 0 to change, 2 to delete, 1 unchanged
+
+>>> uv run [UV_ARGS] -q [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Recreated job_runs.py_run
+Recreated job_runs.yaml_run
+Files: 0 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 2 deleted, 1 unchanged

acceptance/bundle/python/job_runs-on_value_change/resources.py

@@ -0,0 +1,15 @@
+from databricks.bundles.core import Resources
+
+
+def load_resources() -> Resources:
+    resources = Resources()
+
+    resources.add_job_run(
+        "py_run",
+        {
+            "job_id": "${resources.jobs.my_job.id}",
+            "lifecycle": {"triggers": [{"on_value_change": "${var.x}"}]},
+        },
+    )
+
+    return resources

acceptance/bundle/python/job_runs-on_value_change/script

@@ -0,0 +1,20 @@
+title "validate resolves watches from YAML, from Python and from a Python mutator"
+trace uv run $UV_ARGS -q $CLI bundle validate --output json | jq .resources.job_runs
+
+title "first deploy runs both"
+trace uv run $UV_ARGS -q $CLI bundle deploy
+
+title "redeploy with unchanged values plans nothing"
+trace uv run $UV_ARGS -q $CLI bundle plan
+
+title "changing the variable watched in YAML and Python re-fires both runs"
+update_file.py databricks.yml "default: x1" "default: x2"
+trace uv run $UV_ARGS -q $CLI bundle plan
+trace uv run $UV_ARGS -q $CLI bundle deploy
+
+title "changing the variable the Python mutator watches re-fires both runs"
+update_file.py databricks.yml "default: y1" "default: y2"
+trace uv run $UV_ARGS -q $CLI bundle plan
+trace uv run $UV_ARGS -q $CLI bundle deploy
+
+rm -fr .databricks __pycache__

acceptance/bundle/python/job_runs-on_value_change/test.toml

@@ -0,0 +1,7 @@
+Cloud = false # tests don't interact with APIs
+
+# job_runs are only supported in the current version of the wheel
+EnvMatrix.PYDAB_VERSION = ["current"]
+
+# Both runs fire in parallel, so their progress lines interleave in any order.
+SuppressLines.run_output = '^Output from job_runs\.'

acceptance/bundle/refschema/output/job_runs.txt

@@ -19,10 +19,13 @@ resources.job_runs.*.lifecycle.triggers	[]resources.JobRunTrigger	INPUT
 resources.job_runs.*.lifecycle.triggers[*]	resources.JobRunTrigger	INPUT
 resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy	*bool	INPUT
 resources.job_runs.*.lifecycle.triggers[*].on_file_change	*string	INPUT
+resources.job_runs.*.lifecycle.triggers[*].on_value_change	*string	INPUT
 resources.job_runs.*.lifecycle.triggers_state	*resources.JobRunTriggersState	INPUT	STATE
 resources.job_runs.*.lifecycle.triggers_state.on_bundle_deploy	string	INPUT	STATE
 resources.job_runs.*.lifecycle.triggers_state.on_file_change	map[string]string	INPUT	STATE
 resources.job_runs.*.lifecycle.triggers_state.on_file_change.*	string	INPUT	STATE
+resources.job_runs.*.lifecycle.triggers_state.on_value_change	[]string	INPUT	STATE
+resources.job_runs.*.lifecycle.triggers_state.on_value_change[*]	string	INPUT	STATE
 resources.job_runs.*.modified_status	string	INPUT
 resources.job_runs.*.notebook_params	map[string]string	ALL
 resources.job_runs.*.notebook_params.*	string	ALL

acceptance/bundle/resources/job_runs/on_value_change/databricks.yml

@@ -0,0 +1,25 @@
+bundle:
+  name: job-runs-on-value-change
+
+variables:
+  stamp:
+    default: v1
+  other:
+    default: a1
+  number:
+    default: 42
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+  job_runs:
+    my_run:
+      job_id: ${resources.jobs.my_job.id}
+      lifecycle:
+        triggers: [{on_file_change: watched.txt}, {on_value_change: "${var.stamp}"}, {on_value_change: "${var.other}"}, {on_value_change: "${var.number}"}, {on_value_change: "${workspace.root_path}"}]

acceptance/bundle/resources/job_runs/on_value_change/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]