Remove the empty .bundle/<name> directory on destroy

.nextchanges/bundles/destroy-removes-bundle-directory.md

@@ -0,0 +1 @@
+`bundle destroy` now also removes the directory named after the bundle, not just the target directory beneath it, when `workspace.root_path` ends in `${bundle.name}/${bundle.target}` — which includes the default root path. It is removed non-recursively, so it stays in place while another target of the same bundle is still deployed there. Previously every destroy left an empty directory behind ([#6317](https://github.com/databricks/cli/pull/6317)).

acceptance/bundle/destroy/all-resources/output.txt

@@ -21,3 +21,7 @@ Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascad
 All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
 
 Destroy: 2 deleted
+
+=== Assert the bundle directory is deleted
+>>> musterr [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/test-bundle
+Error: Path (//Workspace/Users/[USERNAME]/.bundle/test-bundle) doesn't exist.

acceptance/bundle/destroy/all-resources/script

@@ -1,2 +1,7 @@
 trace $CLI bundle deploy
 trace $CLI bundle destroy --auto-approve
+
+# Destroy removes the target directory, and the .bundle/<name> parent goes with it once
+# nothing else is deployed under it. Double slash keeps Windows from rewriting the path.
+title "Assert the bundle directory is deleted"
+trace musterr $CLI workspace get-status "//Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle"

acceptance/bundle/destroy/root-path-name-target/databricks.yml.tmpl

@@ -0,0 +1,5 @@
+bundle:
+  name: test-bundle
+
+workspace:
+  root_path: ~/.bundle/explicit-$UNIQUE_NAME/${bundle.name}/${bundle.target}

acceptance/bundle/destroy/root-path-name-target/output.txt

@@ -0,0 +1,23 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/explicit-[UNIQUE_NAME]/test-bundle/default/files...
+Files: 5 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/explicit-[UNIQUE_NAME]/test-bundle/default
+
+Destroy: 0 deleted
+
+=== Assert the bundle directory is deleted
+>>> musterr [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/explicit-[UNIQUE_NAME]/test-bundle
+Error: Path (//Workspace/Users/[USERNAME]/.bundle/explicit-[UNIQUE_NAME]/test-bundle) doesn't exist.
+
+=== Assert the prefix above it is kept
+>>> [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/explicit-[UNIQUE_NAME]
+{
+  "object_type": "DIRECTORY"
+}
+
+=== Clean up
+>>> [CLI] workspace delete //Workspace/Users/[USERNAME]/.bundle/explicit-[UNIQUE_NAME] --recursive

acceptance/bundle/destroy/root-path-name-target/script

@@ -0,0 +1,17 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+PREFIX="//Workspace/Users/${CURRENT_USER_NAME}/.bundle/explicit-${UNIQUE_NAME}"
+
+trace $CLI bundle deploy
+trace $CLI bundle destroy --auto-approve
+
+# root_path spells out ${bundle.name}/${bundle.target}, so the directory named after the
+# bundle goes with the deployment even outside the default location.
+title "Assert the bundle directory is deleted"
+trace musterr $CLI workspace get-status "${PREFIX}/test-bundle"
+
+title "Assert the prefix above it is kept"
+trace $CLI workspace get-status "${PREFIX}" | jq '{object_type}'
+
+title "Clean up"
+trace $CLI workspace delete "${PREFIX}" --recursive

acceptance/bundle/destroy/root-path-name-target/test.toml

@@ -0,0 +1,6 @@
+Cloud = true
+
+Ignore = [
+    "databricks.yml",
+    ".databricks/"
+]

acceptance/bundle/destroy/root-path-not-scoped/databricks.yml.tmpl

@@ -0,0 +1,5 @@
+bundle:
+  name: test-bundle
+
+workspace:
+  root_path: ~/.bundle/custom-root-$UNIQUE_NAME/inner

acceptance/bundle/destroy/root-path-not-scoped/output.txt

@@ -0,0 +1,19 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/custom-root-[UNIQUE_NAME]/inner/files...
+Files: 5 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/custom-root-[UNIQUE_NAME]/inner
+
+Destroy: 0 deleted
+
+=== Assert the parent directory is kept
+>>> [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/custom-root-[UNIQUE_NAME]
+{
+  "object_type": "DIRECTORY"
+}
+
+=== Clean up
+>>> [CLI] workspace delete //Workspace/Users/[USERNAME]/.bundle/custom-root-[UNIQUE_NAME] --recursive

acceptance/bundle/destroy/root-path-not-scoped/script

@@ -0,0 +1,14 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+PARENT="//Workspace/Users/${CURRENT_USER_NAME}/.bundle/custom-root-${UNIQUE_NAME}"
+
+trace $CLI bundle deploy
+trace $CLI bundle destroy --auto-approve
+
+# root_path does not end in ${bundle.name}/${bundle.target}, so the directory above it is
+# not this bundle's and is left alone.
+title "Assert the parent directory is kept"
+trace $CLI workspace get-status "${PARENT}" | jq '{object_type}'
+
+title "Clean up"
+trace $CLI workspace delete "${PARENT}" --recursive

acceptance/bundle/destroy/root-path-not-scoped/test.toml

@@ -0,0 +1,6 @@
+Cloud = true
+
+Ignore = [
+    "databricks.yml",
+    ".databricks/"
+]

acceptance/bundle/destroy/sibling-target/databricks.yml.tmpl

@@ -0,0 +1,7 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+targets:
+  dev:
+    default: true
+  prod:

acceptance/bundle/destroy/sibling-target/output.txt

@@ -0,0 +1,32 @@
+
+>>> [CLI] bundle deploy -t dev
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev/files...
+Files: 5 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] bundle deploy -t prod
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/prod/files...
+Files: 5 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Destroy one target while the other is still deployed
+>>> [CLI] bundle destroy -t dev --auto-approve
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev
+
+Destroy: 0 deleted
+
+=== Assert the bundle directory is kept: the sibling target is still under it
+>>> [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]
+{
+  "object_type": "DIRECTORY"
+}
+
+=== Destroy the remaining target
+>>> [CLI] bundle destroy -t prod --auto-approve
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/prod
+
+Destroy: 0 deleted
+
+=== Assert the bundle directory is deleted
+>>> musterr [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]
+Error: Path (//Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]) doesn't exist.

acceptance/bundle/destroy/sibling-target/script

@@ -0,0 +1,19 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+# Double slash keeps Windows from rewriting the path.
+BUNDLE_DIR="//Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle-${UNIQUE_NAME}"
+
+trace $CLI bundle deploy -t dev
+trace $CLI bundle deploy -t prod
+
+title "Destroy one target while the other is still deployed"
+trace $CLI bundle destroy -t dev --auto-approve
+
+title "Assert the bundle directory is kept: the sibling target is still under it"
+trace $CLI workspace get-status "${BUNDLE_DIR}" | jq '{object_type}'
+
+title "Destroy the remaining target"
+trace $CLI bundle destroy -t prod --auto-approve
+
+title "Assert the bundle directory is deleted"
+trace musterr $CLI workspace get-status "${BUNDLE_DIR}"

acceptance/bundle/destroy/sibling-target/test.toml

@@ -0,0 +1,6 @@
+Cloud = true
+
+Ignore = [
+    "databricks.yml",
+    ".databricks/"
+]

acceptance/bundle/resource_deps/remote_app_url/output.txt

@@ -107,6 +107,13 @@ Destroy: 2 deleted
   "method": "DELETE",
   "path": "/api/2.0/pipelines/[UUID]"
 }
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace/delete",
+  "body": {
+    "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle"
+  }
+}
 {
   "method": "POST",
   "path": "/api/2.0/workspace/delete",

acceptance/bundle/user_agent/output.txt

@@ -59,6 +59,7 @@ OK  	destroy.direct	/api/2.0/workspace/get-status	engine/direct
 OK  	destroy.direct	/api/2.1/unity-catalog/schemas/mycatalog.myschema	engine/direct
 OK  	destroy.direct	/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock	engine/direct
 OK  	destroy.direct	/api/2.0/workspace/delete	engine/direct
+OK  	destroy.direct	/api/2.0/workspace/delete	engine/direct
 MISS	destroy.direct	/.well-known/databricks-config	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]'
 MISS	destroy.terraform	/api/2.0/preview/scim/v2/Me	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat'
 MISS	destroy.terraform	/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat'
@@ -70,6 +71,7 @@ OK  	destroy.terraform	/api/2.0/workspace/get-status	engine/terraform
 OK  	destroy.terraform	/api/2.0/workspace/get-status	engine/terraform
 OK  	destroy.terraform	/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock	engine/terraform
 OK  	destroy.terraform	/api/2.0/workspace/delete	engine/terraform
+OK  	destroy.terraform	/api/2.0/workspace/delete	engine/terraform
 MISS	destroy.terraform	/.well-known/databricks-config	'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]'
 MISS	destroy.terraform	/api/2.1/unity-catalog/schemas/mycatalog.myschema	'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat'
 MISS	destroy.terraform	/api/2.1/unity-catalog/current-metastore-assignment	'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat'

acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json

@@ -132,6 +132,18 @@
     "User": "[USERNAME]"
   }
 }
+{
+  "headers": {
+    "User-Agent": [
+      "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat"
+    ]
+  },
+  "method": "POST",
+  "path": "/api/2.0/workspace/delete",
+  "body": {
+    "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle"
+  }
+}
 {
   "headers": {
     "User-Agent": [

acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json

@@ -111,6 +111,18 @@
     "User": "[USERNAME]"
   }
 }
+{
+  "headers": {
+    "User-Agent": [
+      "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none engine/terraform auth/pat"
+    ]
+  },
+  "method": "POST",
+  "path": "/api/2.0/workspace/delete",
+  "body": {
+    "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle"
+  }
+}
 {
   "headers": {
     "User-Agent": [

bundle/bundle.go

@@ -157,6 +157,11 @@ type Bundle struct {
 	// Target stores a snapshot of the Root.Bundle.Target configuration when it was selected by SelectTarget.
 	Target *config.Target `json:"target_config,omitempty" bundle:"internal"`
 
+	// RootPathIsNameTargetScoped reports whether workspace.root_path ends in the bundle
+	// name and target. Recorded before variable resolution, so a path that only happens
+	// to end in those two segments does not count.
+	RootPathIsNameTargetScoped bool
+
 	// Metadata about the bundle deployment. This is the interface Databricks services
 	// rely on to integrate with bundles when they need additional information about
 	// a bundle deployment.

bundle/config/mutator/default_workspace_root.go

@@ -3,11 +3,16 @@ package mutator
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/diag"
 )
 
+// How the bundle name and target appear in a configured root_path, which this mutator
+// sees before variable resolution replaces them.
+const nameTargetSuffix = "${bundle.name}/${bundle.target}"
+
 type defineDefaultWorkspaceRoot struct{}
 
 // DefineDefaultWorkspaceRoot defines the default workspace root path.
@@ -21,6 +26,7 @@ func (m *defineDefaultWorkspaceRoot) Name() string {
 
 func (m *defineDefaultWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	if b.Config.Workspace.RootPath != "" {
+		b.RootPathIsNameTargetScoped = endsWithNameAndTarget(b.Config.Workspace.RootPath)
 		return nil
 	}
 
@@ -37,5 +43,13 @@ func (m *defineDefaultWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle
 		b.Config.Bundle.Name,
 		b.Config.Bundle.Target,
 	)
+	b.RootPathIsNameTargetScoped = true
 	return nil
 }
+
+// endsWithNameAndTarget reports whether rootPath ends in the bundle name and target
+// references. Matched as a plain string: rootPath is not interpolated yet, so path
+// operations on it are not reliable.
+func endsWithNameAndTarget(rootPath string) bool {
+	return strings.HasSuffix(strings.TrimSuffix(rootPath, "/"), nameTargetSuffix)
+}

bundle/config/mutator/default_workspace_root_test.go

@@ -24,3 +24,38 @@ func TestDefaultWorkspaceRoot(t *testing.T) {
 
 	assert.Equal(t, "~/.bundle/name/environment", b.Config.Workspace.RootPath)
 }
+
+func TestDefaultWorkspaceRootIsNameTargetScoped(t *testing.T) {
+	tcases := []struct {
+		name     string
+		rootPath string
+		scoped   bool
+	}{
+		{"defaulted", "", true},
+		{"name and target references", "~/.bundle/${bundle.name}/${bundle.target}", true},
+		{"references under another prefix", "/Workspace/Shared/${bundle.name}/${bundle.target}", true},
+		{"trailing slash", "~/.bundle/${bundle.name}/${bundle.target}/", true},
+		// Already-resolved segments are indistinguishable from a literal path that
+		// happens to match, so they do not count.
+		{"resolved values", "~/.bundle/name/environment", false},
+		{"target only", "~/.bundle/${bundle.target}", false},
+		{"name only", "~/.bundle/${bundle.name}", false},
+		{"extra segment below", "~/.bundle/${bundle.name}/${bundle.target}/inner", false},
+		{"unrelated path", "/Workspace/Shared/some/path", false},
+	}
+
+	for _, tc := range tcases {
+		t.Run(tc.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				Config: config.Root{
+					Bundle:    config.Bundle{Name: "name", Target: "environment"},
+					Workspace: config.Workspace{RootPath: tc.rootPath},
+				},
+			}
+			diags := bundle.Apply(t.Context(), b, mutator.DefineDefaultWorkspaceRoot())
+			require.NoError(t, diags.Error())
+
+			assert.Equal(t, tc.scoped, b.RootPathIsNameTargetScoped)
+		})
+	}
+}

bundle/deploy/files/delete.go

@@ -6,9 +6,11 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
+	"path"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/sync"
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 )
@@ -28,6 +30,8 @@ func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 		return diag.FromErr(err)
 	}
 
+	removeBundleNameDir(ctx, b)
+
 	// Clean up sync snapshot file
 	err = deleteSnapshotFile(ctx, b)
 	if err != nil {
@@ -36,6 +40,21 @@ func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	return nil
 }
 
+// removeBundleNameDir removes the directory named after the bundle now that the
+// deployment under it is gone. Not recursive, so it fails harmlessly while another
+// target is still deployed there.
+func removeBundleNameDir(ctx context.Context, b *bundle.Bundle) {
+	if !b.RootPathIsNameTargetScoped {
+		return
+	}
+
+	dir := path.Dir(b.Config.Workspace.RootPath)
+	err := b.WorkspaceClient(ctx).Workspace.Delete(ctx, workspace.Delete{Path: dir})
+	if err != nil {
+		log.Infof(ctx, "Leaving %s in place: %s", dir, err)
+	}
+}
+
 func deleteSnapshotFile(ctx context.Context, b *bundle.Bundle) error {
 	opts, err := GetSyncOptions(ctx, b)
 	if err != nil {