From 64ee8842a1c6ec01489f3f4c3493629556f2957c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 24 Aug 2026 05:46:35 +0000 Subject: [PATCH 1/2] acc: remove the bundle directories left behind after cloud test runs The suite-level cleanup added in #6092 destroys every deployment this run leaked, but `bundle destroy` only removes the deployment root under ~/.bundle/ and never the directory itself. Every run picks a fresh bundle name, so nothing reuses those directories and they pile up against the workspace child-node limit -- the CLI test workspace reached 48k+ nodes that way and had to be allow-listed to 100k (ES-2138924). Remove them in the same sweep, right after the destroys. The delete is recursive, so a directory a killed test left half-written goes too, even when no deployment root was found under it. A directory whose destroy failed is kept, so its state is still available to the periodic sweep that reclaims it. Co-authored-by: Isaac --- acceptance/bundle_clean_test.go | 48 ++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle_clean_test.go b/acceptance/bundle_clean_test.go index bb1392173f1..8c161c65e41 100644 --- a/acceptance/bundle_clean_test.go +++ b/acceptance/bundle_clean_test.go @@ -71,15 +71,17 @@ func cleanBundles(ctx context.Context, t *testing.T, execPath, prefix string) { // (the bundle name, or the leaf of a workspace.root_path override), so match // there and only descend into this run's subtrees. This avoids walking the // thousands of directories other runs may have leaked under .bundle. - var roots []string + var roots, bundleDirs []string for _, bundleRoot := range bundleRoots { for _, child := range listChildDirs(ctx, t, w, bundleRoot) { if strings.Contains(path.Base(child), prefix) { + bundleDirs = append(bundleDirs, child) roots = append(roots, findDeploymentRoots(ctx, t, w, child)...) } } } slices.Sort(roots) + slices.Sort(bundleDirs) t.Logf("%s bundle cleanup: found %d deployment(s) with prefix %q", time.Now().Format(time.RFC3339), len(roots), prefix) @@ -122,6 +124,50 @@ func cleanBundles(ctx context.Context, t *testing.T, execPath, prefix string) { if len(failed) > 0 { t.Logf("WARNING: bundle cleanup failed to destroy %d deployment(s), leaked until swept: %s", len(failed), strings.Join(failed, ", ")) } + + removeBundleDirs(ctx, t, w, bundleDirs, failed) +} + +// removeBundleDirs deletes the ~/.bundle/ directories this run created. +// Destroy removes the deployment root beneath such a directory but never the +// directory itself, and every run picks a fresh bundle name, so nothing reuses +// them: left behind they pile up against the workspace child-node limit, which +// is what exhausted it before. The delete is recursive so a directory a killed +// test left half-written goes too. A directory whose deployment failed to +// destroy is kept, so its state is still there for the periodic sweep. +func removeBundleDirs(ctx context.Context, t *testing.T, w *databricks.WorkspaceClient, bundleDirs, failed []string) { + const maxConcurrentDeletes = 20 + sem := make(chan struct{}, maxConcurrentDeletes) + var wg sync.WaitGroup + var mu sync.Mutex + var failedRemovals, kept []string + for _, dir := range bundleDirs { + if slices.ContainsFunc(failed, func(root string) bool { + return root == dir || strings.HasPrefix(root, dir+"/") + }) { + kept = append(kept, dir) + continue + } + sem <- struct{}{} + wg.Go(func() { + defer func() { <-sem }() + err := w.Workspace.Delete(ctx, workspace.Delete{Path: dir, Recursive: true}) + if err != nil && !errors.Is(err, apierr.ErrNotFound) { + mu.Lock() + failedRemovals = append(failedRemovals, dir) + mu.Unlock() + } + }) + } + wg.Wait() + + t.Logf("%s bundle cleanup: removed %d/%d bundle directory(ies), kept %d holding a failed destroy", + time.Now().Format(time.RFC3339), len(bundleDirs)-len(kept)-len(failedRemovals), len(bundleDirs)-len(kept), len(kept)) + + if len(failedRemovals) > 0 { + slices.Sort(failedRemovals) + t.Logf("WARNING: bundle cleanup could not remove %d bundle directory(ies): %s", len(failedRemovals), strings.Join(failedRemovals, ", ")) + } } // findDeploymentRoots walks the workspace tree under dir and returns the paths From 15ab7cb23b3e836ee6cff6e4503deef2a51d223d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 24 Aug 2026 06:16:49 +0000 Subject: [PATCH 2/2] Shorten the removeBundleDirs comment Co-authored-by: Isaac --- acceptance/bundle_clean_test.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle_clean_test.go b/acceptance/bundle_clean_test.go index 8c161c65e41..6f64e4f53f9 100644 --- a/acceptance/bundle_clean_test.go +++ b/acceptance/bundle_clean_test.go @@ -129,12 +129,8 @@ func cleanBundles(ctx context.Context, t *testing.T, execPath, prefix string) { } // removeBundleDirs deletes the ~/.bundle/ directories this run created. -// Destroy removes the deployment root beneath such a directory but never the -// directory itself, and every run picks a fresh bundle name, so nothing reuses -// them: left behind they pile up against the workspace child-node limit, which -// is what exhausted it before. The delete is recursive so a directory a killed -// test left half-written goes too. A directory whose deployment failed to -// destroy is kept, so its state is still there for the periodic sweep. +// Destroy empties them but leaves them behind, and the names are never reused, so +// they pile up in the workspace. One whose destroy failed is left for the sweep. func removeBundleDirs(ctx context.Context, t *testing.T, w *databricks.WorkspaceClient, bundleDirs, failed []string) { const maxConcurrentDeletes = 20 sem := make(chan struct{}, maxConcurrentDeletes)