Add lifecycle.bind to adopt existing workspace resources at deploy time

.nextchanges/bundles/4630.md

@@ -0,0 +1 @@
+* Add `lifecycle.bind` to resources to bring an existing workspace resource under bundle management at deploy time instead of creating a new one. The first deploy adopts the resource (`bind`) and applies the configuration if it differs (`bind_and_update`); while the block is present, deploys cannot recreate the resource, and `bundle destroy` still deletes it. ([#4630](https://github.com/databricks/cli/pull/4630), [#7321](https://github.com/databricks/cli/pull/7321))

acceptance/bundle/deploy/bind/basic/databricks.yml

@@ -0,0 +1,19 @@
+bundle:
+  name: test-bind-basic-UNIQUE_NAME
+
+resources:
+  jobs:
+    foo:
+      name: test-bind-job
+      environments:
+        - environment_key: default
+          spec:
+            client: "1"
+      tasks:
+        - task_key: my_task
+          environment_key: default
+          spark_python_task:
+            python_file: ./hello.py
+      lifecycle:
+        bind:
+          id: "PLACEHOLDER_JOB_ID"

acceptance/bundle/deploy/bind/basic/hello.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/deploy/bind/basic/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deploy/bind/basic/output.txt

@@ -0,0 +1,65 @@
+
+>>> [CLI] bundle plan
+bind_and_update jobs.foo (id: [NEW_JOB_ID])
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged, 1 to bind
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-basic-[UNIQUE_NAME]/default/files...
+Bound and updated jobs.foo
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged, 1 bound
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> print_state.py
+{
+ "state_version": 3,
+ "cli_version": "[CLI_VERSION]",
+ "lineage": "[UUID]",
+ "serial": 1,
+ "state": {
+  "resources.jobs.foo": {
+   "__id__": "[NEW_JOB_ID]",
+   "state": {
+    "deployment": {
+     "kind": "BUNDLE",
+     "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bind-basic-[UNIQUE_NAME]/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "environments": [
+     {
+      "environment_key": "default",
+      "spec": {
+       "client": "1"
+      }
+     }
+    ],
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "test-bind-job",
+    "queue": {
+     "enabled": true
+    },
+    "tasks": [
+     {
+      "environment_key": "default",
+      "spark_python_task": {
+       "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bind-basic-[UNIQUE_NAME]/default/files/hello.py"
+      },
+      "task_key": "my_task"
+     }
+    ]
+   }
+  }
+ }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-basic-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/deploy/bind/basic/script

@@ -0,0 +1,25 @@
+# Bundle names carry $UNIQUE_NAME so concurrent runs on the same workspace do not share
+# a root path and its deployment lock.
+update_file.py databricks.yml 'UNIQUE_NAME' "$UNIQUE_NAME"
+
+# Create a job in the workspace
+NEW_JOB_ID=$($CLI jobs create --json '{"name": "test-import-job", "environments": [{"environment_key": "default", "spec": {"client": "1"}}], "tasks": [{"task_key": "my_task", "environment_key": "default", "spark_python_task": {"python_file": "/Workspace/test.py"}}]}' | jq -r .job_id)
+add_repl.py $NEW_JOB_ID NEW_JOB_ID
+
+# Update the databricks.yml with the actual job ID
+update_file.py databricks.yml 'PLACEHOLDER_JOB_ID' "$NEW_JOB_ID"
+
+# Run plan - should show import action
+trace $CLI bundle plan
+
+# Deploy with auto-approve
+trace $CLI bundle deploy --auto-approve
+
+# Plan again - should show no changes (skip)
+trace $CLI bundle plan
+
+# Verify state file contains the imported ID
+trace print_state.py | contains.py "$NEW_JOB_ID"
+
+# Cleanup: destroy is allowed even with an active bind block
+trace $CLI bundle destroy --auto-approve

acceptance/bundle/deploy/bind/bind-and-resize/databricks.yml.tmpl

@@ -0,0 +1,16 @@
+bundle:
+  name: test-bind-resize
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  clusters:
+    test_cluster:
+      cluster_name: test-cluster-$UNIQUE_NAME
+      spark_version: $DEFAULT_SPARK_VERSION
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
+      num_workers: 3
+      lifecycle:
+        bind:
+          id: "PLACEHOLDER_CLUSTER_ID"

acceptance/bundle/deploy/bind/bind-and-resize/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deploy/bind/bind-and-resize/output.txt

@@ -0,0 +1,37 @@
+
+>>> [CLI] bundle plan
+bind_and_update clusters.test_cluster (id: [CLUSTER_ID])
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged, 1 to bind
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Bound and updated clusters.test_cluster
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged, 1 bound
+
+>>> print_requests.py //clusters/resize //clusters/edit
+{
+  "method": "POST",
+  "path": "/api/2.1/clusters/resize",
+  "body": {
+    "cluster_id": "[CLUSTER_ID]",
+    "num_workers": 3
+  }
+}
+
+=== Cluster should have new num_workers
+
+>>> [CLI] clusters get [CLUSTER_ID]
+{
+  "cluster_name": "test-cluster-[UNIQUE_NAME]",
+  "num_workers": 3
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.clusters.test_cluster
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 1 deleted

acceptance/bundle/deploy/bind/bind-and-resize/script

@@ -0,0 +1,26 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+# Create a cluster out-of-band, then bring it to RUNNING so that a num_workers
+# change is planned as a resize rather than an edit. The cluster uses an instance
+# pool only: the API rejects node_type_id together with instance_pool_id.
+CLUSTER_ID=$($CLI clusters create --json "{\"cluster_name\": \"test-cluster-$UNIQUE_NAME\", \"spark_version\": \"$DEFAULT_SPARK_VERSION\", \"instance_pool_id\": \"$TEST_INSTANCE_POOL_ID\", \"num_workers\": 2, \"autotermination_minutes\": 60}" | jq -r .cluster_id)
+add_repl.py "$CLUSTER_ID" CLUSTER_ID
+update_file.py databricks.yml 'PLACEHOLDER_CLUSTER_ID' "$CLUSTER_ID"
+
+# Transition PENDING -> RUNNING (ClustersGet promotes the state).
+$CLI clusters get "$CLUSTER_ID" > /dev/null
+
+# Plan: config differs only in num_workers, so the bound running cluster is
+# planned as bind_and_update backed by a resize.
+trace $CLI bundle plan
+
+# Deploy: bind_and_update must go through the resize API, not clusters/edit.
+trace $CLI bundle deploy --auto-approve
+trace print_requests.py //clusters/resize //clusters/edit
+
+title "Cluster should have new num_workers\n"
+trace $CLI clusters get "$CLUSTER_ID" | jq '{cluster_name,num_workers}'
+
+# Cleanup
+trace $CLI bundle destroy --auto-approve
+rm -f out.requests.txt

acceptance/bundle/deploy/bind/bind-and-resize/test.toml

@@ -0,0 +1,3 @@
+RecordRequests = true
+
+Ignore = [".databricks", "databricks.yml"]

acceptance/bundle/deploy/bind/bind-and-update/databricks.yml

@@ -0,0 +1,19 @@
+bundle:
+  name: test-bind-update-UNIQUE_NAME
+
+resources:
+  jobs:
+    foo:
+      name: updated-job-name
+      environments:
+        - environment_key: default
+          spec:
+            client: "1"
+      tasks:
+        - task_key: my_task
+          environment_key: default
+          spark_python_task:
+            python_file: ./hello.py
+      lifecycle:
+        bind:
+          id: "PLACEHOLDER_JOB_ID"

acceptance/bundle/deploy/bind/bind-and-update/hello.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/deploy/bind/bind-and-update/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deploy/bind/bind-and-update/output.txt

@@ -0,0 +1,36 @@
+
+>>> [CLI] bundle plan
+bind_and_update jobs.foo (id: [NEW_JOB_ID])
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged, 1 to bind
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-update-[UNIQUE_NAME]/default/files...
+Bound and updated jobs.foo
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged, 1 bound
+
+>>> [CLI] jobs get [NEW_JOB_ID]
+updated-job-name
+
+>>> [CLI] bundle plan
+update jobs.foo
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-update-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] jobs get [NEW_JOB_ID]
+second-update-name
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-update-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/deploy/bind/bind-and-update/script

@@ -0,0 +1,30 @@
+update_file.py databricks.yml 'UNIQUE_NAME' "$UNIQUE_NAME"
+
+# Create a job in the workspace with a different name
+NEW_JOB_ID=$($CLI jobs create --json '{"name": "original-job-name", "environments": [{"environment_key": "default", "spec": {"client": "1"}}], "tasks": [{"task_key": "my_task", "environment_key": "default", "spark_python_task": {"python_file": "/Workspace/test.py"}}]}' | jq -r .job_id)
+add_repl.py $NEW_JOB_ID NEW_JOB_ID
+
+# Update the databricks.yml with the actual job ID
+update_file.py databricks.yml 'PLACEHOLDER_JOB_ID' "$NEW_JOB_ID"
+
+# Run plan - should show bind action (name differs from config)
+trace $CLI bundle plan
+
+# Deploy with auto-approve
+trace $CLI bundle deploy --auto-approve
+
+# Verify the job was updated
+trace $CLI jobs get $NEW_JOB_ID | jq -r .settings.name
+
+# Now update the job name again in the config and deploy again.
+# This time the action should be "update", not "bind", since the resource
+# is already bound in state.
+update_file.py databricks.yml 'updated-job-name' 'second-update-name'
+trace $CLI bundle plan
+trace $CLI bundle deploy --auto-approve
+
+# Verify the job was updated with the second name
+trace $CLI jobs get $NEW_JOB_ID | jq -r .settings.name
+
+# Cleanup: destroy is allowed even with an active bind block
+trace $CLI bundle destroy --auto-approve

acceptance/bundle/deploy/bind/block-removed/databricks.yml

@@ -0,0 +1,14 @@
+bundle:
+  name: test-bind-block-removed-UNIQUE_NAME
+
+resources:
+  pipelines:
+    foo:
+      name: test-pipeline
+      storage: /old/storage/path
+      libraries:
+        - notebook:
+            path: ./nb.sql
+      lifecycle:
+        bind:
+          id: "PLACEHOLDER_PIPELINE_ID"

acceptance/bundle/deploy/bind/block-removed/databricks_unbound.yml

@@ -0,0 +1,11 @@
+bundle:
+  name: test-bind-block-removed-UNIQUE_NAME
+
+resources:
+  pipelines:
+    foo:
+      name: test-pipeline
+      storage: /new/storage/path
+      libraries:
+        - notebook:
+            path: ./nb.sql

acceptance/bundle/deploy/bind/block-removed/nb.sql

@@ -0,0 +1,2 @@
+-- Databricks notebook source
+select 1

acceptance/bundle/deploy/bind/block-removed/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/deploy/bind/block-removed/output.txt

@@ -0,0 +1,24 @@
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-block-removed-[UNIQUE_NAME]/default/files...
+Bound and updated pipelines.foo
+Files: 3 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged, 1 bound
+
+=== Removing the bind block is the opt-out: with it gone, the same storage change plans a recreate
+>>> [CLI] bundle plan
+recreate pipelines.foo
+
+Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.pipelines.foo
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:
+  delete resources.pipelines.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-block-removed-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/deploy/bind/block-removed/script

@@ -0,0 +1,16 @@
+update_file.py databricks.yml 'UNIQUE_NAME' "$UNIQUE_NAME"
+update_file.py databricks_unbound.yml 'UNIQUE_NAME' "$UNIQUE_NAME"
+
+# Create a pipeline whose storage matches the config and bind it
+NEW_PIPELINE_ID=$($CLI pipelines create --json '{"name": "test-pipeline", "storage": "/old/storage/path", "allow_duplicate_names": true, "libraries": [{"notebook": {"path": "/Workspace/test"}}]}' | jq -r .pipeline_id)
+add_repl.py $NEW_PIPELINE_ID NEW_PIPELINE_ID
+update_file.py databricks.yml 'PLACEHOLDER_PIPELINE_ID' "$NEW_PIPELINE_ID"
+
+trace $CLI bundle deploy --auto-approve
+
+title "Removing the bind block is the opt-out: with it gone, the same storage change plans a recreate"
+cp databricks_unbound.yml databricks.yml
+trace $CLI bundle plan
+
+# Cleanup
+trace $CLI bundle destroy --auto-approve

acceptance/bundle/deploy/bind/block-removed/test.toml

@@ -0,0 +1 @@
+Ignore = [".databricks", "databricks_unbound.yml"]

acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: test-bind-delete-conflict-UNIQUE_NAME
+
+resources:
+  jobs:
+    foo:
+      name: test-bind-delete-job
+      environments:
+        - environment_key: default
+          spec:
+            client: "1"
+      tasks:
+        - task_key: my_task
+          environment_key: default
+          spark_python_task:
+            python_file: ./hello.py