Record deployment history for bundle resource unbindings

.nextchanges/bundles/record-unbind-history.md

@@ -0,0 +1 @@
+* Record deployment history when unbinding resources with `bundle deployment unbind` when deployment history is enabled. ([#7401](https://github.com/databricks/cli/pull/7401))

acceptance/bundle/dms/bind-unbind/databricks.yml.tmpl

@@ -4,3 +4,6 @@ resources:
   jobs:
     foo:
       name: foo
+      permissions:
+        - group_name: users
+          level: CAN_VIEW

acceptance/bundle/dms/bind-unbind/output.txt

@@ -41,9 +41,10 @@ Run 'bundle deploy' to deploy changes to your workspace
 === Deploy keeps the bound job ID with recorded history
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files...
+Created jobs.foo.permissions
 Updated jobs.foo
 Files: 2 uploaded, 0 deleted
-Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+Resources: 1 created, 1 changed, 0 deleted, 0 unchanged
 
 >>> jq {features, resource_id: .state["resources.jobs.foo"].__id__} .databricks/bundle/default/resources.json
 {
@@ -56,6 +57,12 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
 === Unbind preserves the shadow history marker and leaves the job intact
 >>> [CLI] bundle deployment unbind foo
 
+>>> print_requests.py --dms //versions //jobs //permissions --oneline
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "5"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UNBIND", "resource_key": "jobs.foo"}, {"action_type": "OPERATION_ACTION_TYPE_UNBIND", "resource_key": "jobs.foo.permissions"}], "previous_version_id": "4", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/5/operations/jobs.foo", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/5/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/5/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+
 >>> jq {features, state} .databricks/bundle/default/resources.json
 {
   "features": {
@@ -69,6 +76,14 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
   "job_id": [JOB_ID]
 }
 
+>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json
+[]
+
+=== Unbinding an already unbound resource creates no history version
+>>> [CLI] bundle deployment unbind foo
+
+>>> print_requests.py --dms //versions
+
 === Rebind preserves the existing shadow history marker
 >>> [CLI] bundle deployment bind foo [JOB_ID] --auto-approve
 Successfully bound job with an id '[JOB_ID]'
@@ -83,15 +98,16 @@ Run 'bundle deploy' to deploy changes to your workspace
 }
 
 >>> print_requests.py --dms //versions --oneline
-{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "7"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_BIND", "resource_key": "jobs.foo"}], "previous_version_id": "4", "version_type": "VERSION_TYPE_DEPLOY"}}
-{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/7/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[JOB_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
-{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/7/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "8"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_BIND", "resource_key": "jobs.foo"}], "previous_version_id": "5", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/8/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[JOB_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/8/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
 
 === A subsequent deploy continues managing the same job
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files...
+Created jobs.foo.permissions
 Files: 0 uploaded, 0 deleted
-Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
+Resources: 1 created, 0 changed, 0 deleted, 1 unchanged
 
 >>> jq {features, resource_id: .state["resources.jobs.foo"].__id__} .databricks/bundle/default/resources.json
 {

acceptance/bundle/dms/bind-unbind/script

@@ -23,13 +23,22 @@ title "Deploy keeps the bound job ID with recorded history"
 trace $CLI bundle deploy
 trace jq '{features, resource_id: .state["resources.jobs.foo"].__id__}' .databricks/bundle/default/resources.json
 
+rm -f "$OUT_REQUESTS"
+
 title "Unbind preserves the shadow history marker and leaves the job intact"
 trace $CLI bundle deployment unbind foo
+trace print_requests.py --dms //versions //jobs //permissions --oneline
 trace jq '{features, state}' .databricks/bundle/default/resources.json
 trace $CLI jobs get "$job_id" | jq '{job_id}'
+trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, resource_id, last_action_type})'
+
 
 rm -f "$OUT_REQUESTS"
 
+title "Unbinding an already unbound resource creates no history version"
+trace $CLI bundle deployment unbind foo
+trace print_requests.py --dms //versions
+
 title "Rebind preserves the existing shadow history marker"
 trace $CLI bundle deployment bind foo "$job_id" --auto-approve
 trace jq '{features, resource_id: .state["resources.jobs.foo"].__id__}' .databricks/bundle/default/resources.json

acceptance/bundle/dms/unbind-history-failures/databricks.yml.tmpl

@@ -0,0 +1,13 @@
+bundle:
+  name: dms-unbind-history-failures-$UNIQUE_NAME
+
+resources:
+  jobs:
+    create_failed:
+      name: create_failed
+    record_failed:
+      name: record_failed
+    complete_failed:
+      name: complete_failed
+    upload_failed:
+      name: upload_failed

acceptance/bundle/dms/unbind-history-failures/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["true"]

acceptance/bundle/dms/unbind-history-failures/output.txt

@@ -0,0 +1,80 @@
+
+=== A version creation failure does not prevent unbinding
+>>> fault.py POST /api/2.0/bundle/deployments/*/versions 400 0 1
+
+>>> [CLI] bundle deployment unbind create_failed
+Warn: Failed to record unbind history: failed to create deployment version: Fault injected by test.
+
+=== An operation recording failure does not prevent unbinding
+>>> fault.py PATCH /api/2.0/bundle/** 400 0 1
+
+>>> [CLI] bundle deployment unbind record_failed
+Warn: Failed to record unbind history: Fault injected by test.
+
+=== A version completion failure does not prevent unbinding
+>>> fault.py POST /api/2.0/bundle/deployments/*/versions/*/complete 400 0 1
+
+>>> [CLI] bundle deployment unbind complete_failed
+Warn: Failed to complete deployment history version: Fault injected by test.
+
+=== Successful unbindings survived the remote state round trips and the job still exists
+>>> jq .state .databricks/bundle/default/resources.json
+{
+  "resources.jobs.upload_failed": {
+    "__id__": "[JOB_ID]",
+    "state": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-unbind-history-failures-[UNIQUE_NAME]/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "upload_failed",
+      "queue": {
+        "enabled": true
+      }
+    }
+  }
+}
+
+>>> [CLI] jobs get [JOB_ID]
+{
+  "job_id": [JOB_ID],
+  "name": "unbind-target-[UNIQUE_NAME]"
+}
+
+>>> print_requests.py --dms //versions --oneline
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "10"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UNBIND", "resource_key": "jobs.create_failed"}], "previous_version_id": "9", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "11"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UNBIND", "resource_key": "jobs.record_failed"}], "previous_version_id": "9", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/11/operations/jobs.record_failed", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/11/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "12"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UNBIND", "resource_key": "jobs.complete_failed"}], "previous_version_id": "11", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/12/operations/jobs.complete_failed", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/12/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}
+
+=== A remote state upload failure fails the unbind version
+>>> fault.py POST /api/2.0/workspace-files/import-file/**resources.json 400 0 1
+
+>>> musterr [CLI] bundle deployment unbind upload_failed
+Error: Fault injected by test. (400 INJECTED)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Fdms-unbind-history-failures-[UNIQUE_NAME]%2Fdefault%2Fstate%2Fresources.json?overwrite=true
+HTTP Status: 400 Bad Request
+API error_code: INJECTED
+API message: Fault injected by test.
+
+
+>>> print_requests.py --dms //versions --oneline
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "13"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UNBIND", "resource_key": "jobs.upload_failed"}], "previous_version_id": "12", "version_type": "VERSION_TYPE_DEPLOY"}}
+{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/13/operations/jobs.upload_failed", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}}
+{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/13/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}}
+
+=== Remote state still contains the binding whose upload failed
+>>> [CLI] bundle summary -o json
+{
+  "complete_failed": null,
+  "create_failed": null,
+  "record_failed": null,
+  "upload_failed": "[JOB_ID]"
+}

acceptance/bundle/dms/unbind-history-failures/script

@@ -0,0 +1,37 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+job_id=$($CLI jobs create --json '{"name": "unbind-target-'$UNIQUE_NAME'"}' | jq -r '.job_id')
+add_repl "$job_id" JOB_ID
+for key in create_failed record_failed complete_failed upload_failed; do
+    $CLI bundle deployment bind "$key" "$job_id" --auto-approve >> LOG.setup 2>&1
+done
+rm -f "$OUT_REQUESTS"
+
+title "A version creation failure does not prevent unbinding"
+trace fault.py "POST /api/2.0/bundle/deployments/*/versions" 400 0 1
+trace $CLI bundle deployment unbind create_failed
+rm -rf .databricks
+
+title "An operation recording failure does not prevent unbinding"
+trace fault.py "PATCH /api/2.0/bundle/**" 400 0 1
+trace $CLI bundle deployment unbind record_failed
+rm -rf .databricks
+
+title "A version completion failure does not prevent unbinding"
+trace fault.py "POST /api/2.0/bundle/deployments/*/versions/*/complete" 400 0 1
+trace $CLI bundle deployment unbind complete_failed
+
+title "Successful unbindings survived the remote state round trips and the job still exists"
+trace jq '.state' .databricks/bundle/default/resources.json
+trace $CLI jobs get "$job_id" | jq '{job_id, name: .settings.name}'
+trace print_requests.py --dms //versions --oneline
+
+
+title "A remote state upload failure fails the unbind version"
+trace fault.py "POST /api/2.0/workspace-files/import-file/**resources.json" 400 0 1
+trace musterr $CLI bundle deployment unbind upload_failed
+trace print_requests.py --dms //versions --oneline
+
+rm -rf .databricks
+title "Remote state still contains the binding whose upload failed"
+trace $CLI bundle summary -o json | jq '.resources.jobs | map_values(.id // null)'
+rm -f "$OUT_REQUESTS"

acceptance/bundle/dms/unbind-history-failures/test.toml

@@ -0,0 +1 @@
+Cloud = false # Inject failures into the deployment history service.

bundle/direct/bind.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"slices"
 	"strings"
 
 	"github.com/databricks/cli/bundle/config"
@@ -42,6 +43,12 @@ type DMSBindMetadata struct {
 	ResourceState dstate.ResourceEntry
 }
 
+// DMSUnbindMetadata holds the removed resource keys and serial to publish after unbind is committed.
+type DMSUnbindMetadata struct {
+	Serial       int
+	ResourceKeys []string
+}
+
 // BindResult contains the result of a bind operation including any detected changes.
 type BindResult struct {
 	// DMSMetadata is nil when deployment history is disabled.
@@ -260,18 +267,35 @@ func (result *BindResult) Cancel() {
 
 // Unbind removes a resource from direct engine state without deleting
 // the workspace resource. Also removes associated permissions/grants entries.
-func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error {
+// It returns nil DMS metadata when deployment history is disabled.
+func (b *DeploymentBundle) Unbind(ctx context.Context, configRoot *config.Root, statePath, resourceKey string) (*DMSUnbindMetadata, error) {
 	if err := b.StateDB.Open(ctx, statePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil {
-		return err
+		return nil, err
 	}
 	if err := b.StateDB.UpgradeToWrite(); err != nil {
-		return err
+		return nil, err
+	}
+
+	var configured *bool
+	if configRoot.Experimental != nil {
+		configured = configRoot.Experimental.DeploymentHistory
+	}
+	var metadata *DMSUnbindMetadata
+	if b.StateDB.RecordsHistory() || bundleenv.RecordsDeploymentHistory(ctx, configured) {
+		metadata = &DMSUnbindMetadata{Serial: b.StateDB.Data.Serial}
+		// Finalize commits the removals below as one WAL transaction.
+		if b.StateDB.Data.State != nil {
+			metadata.Serial++
+		}
+		if b.StateDB.GetResourceID(resourceKey) != "" {
+			metadata.ResourceKeys = append(metadata.ResourceKeys, resourceKey)
+		}
 	}
 
 	// Delete the main resource
 	err := b.StateDB.DeleteState(ctx, resourceKey, false)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	log.Infof(ctx, "Unbound %s", resourceKey)
@@ -285,12 +309,18 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st
 		if key == permissionsKey || key == grantsKey || strings.HasPrefix(key, resourceKey+".") {
 			err = b.StateDB.DeleteState(ctx, key, false)
 			if err != nil {
-				return err
+				return nil, err
+			}
+			if metadata != nil {
+				metadata.ResourceKeys = append(metadata.ResourceKeys, key)
 			}
 			log.Infof(ctx, "Unbound %s", key)
 		}
 	}
 
+	if metadata != nil {
+		slices.Sort(metadata.ResourceKeys)
+	}
 	_, err = b.StateDB.Finalize(ctx)
-	return err
+	return metadata, err
 }

bundle/direct/bind_test.go

@@ -0,0 +1,58 @@
+package direct_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/direct"
+	"github.com/databricks/cli/bundle/direct/dstate"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUnbindReturnsRemovedKeys(t *testing.T) {
+	for _, tc := range []struct {
+		name    string
+		keys    []string
+		history bool
+	}{
+		{"resource", []string{"resources.jobs.foo"}, true},
+		{"children", []string{"resources.jobs.foo", "resources.jobs.foo.permissions", "resources.jobs.foo.grants"}, true},
+		{"children_without_parent", []string{"resources.jobs.foo.permissions"}, true},
+		{"missing", nil, true},
+		{"history_disabled", []string{"resources.jobs.foo"}, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := t.Context()
+			statePath := filepath.Join(t.TempDir(), "resources.json")
+			var b direct.DeploymentBundle
+			require.NoError(t, b.StateDB.Open(ctx, statePath, dstate.ReadingFile, false, dstate.OpenDmsArgs{}))
+			require.NoError(t, b.StateDB.UpgradeToWrite())
+			for _, key := range append(tc.keys, "resources.jobs.foobar") {
+				require.NoError(t, b.StateDB.SaveState(ctx, key, "id", map[string]string{"name": key}, nil))
+			}
+			_, err := b.StateDB.Finalize(ctx)
+			require.NoError(t, err)
+
+			root := &config.Root{Experimental: &config.Experimental{DeploymentHistory: &tc.history}}
+			metadata, err := b.Unbind(ctx, root, statePath, "resources.jobs.foo")
+			require.NoError(t, err)
+			if tc.history {
+				require.NotNil(t, metadata)
+				assert.ElementsMatch(t, tc.keys, metadata.ResourceKeys)
+			} else {
+				assert.Nil(t, metadata)
+			}
+
+			require.NoError(t, b.StateDB.Open(ctx, statePath, dstate.ReadingFile, false, dstate.OpenDmsArgs{}))
+			assert.Len(t, b.StateDB.Data.State, 1)
+			if metadata != nil {
+				assert.Equal(t, b.StateDB.Data.Serial, metadata.Serial)
+			}
+			assert.Equal(t, "id", b.StateDB.GetResourceID("resources.jobs.foobar"))
+			_, err = b.StateDB.Finalize(ctx)
+			require.NoError(t, err)
+		})
+	}
+}