diff --git a/acceptance/bundle_clean_test.go b/acceptance/bundle_clean_test.go index bb1392173f..6f64e4f53f 100644 --- a/acceptance/bundle_clean_test.go +++ b/acceptance/bundle_clean_test.go @@ -71,15 +71,17 @@ func cleanBundles(ctx context.Context, t *testing.T, execPath, prefix string) { // (the bundle name, or the leaf of a workspace.root_path override), so match // there and only descend into this run's subtrees. This avoids walking the // thousands of directories other runs may have leaked under .bundle. - var roots []string + var roots, bundleDirs []string for _, bundleRoot := range bundleRoots { for _, child := range listChildDirs(ctx, t, w, bundleRoot) { if strings.Contains(path.Base(child), prefix) { + bundleDirs = append(bundleDirs, child) roots = append(roots, findDeploymentRoots(ctx, t, w, child)...) } } } slices.Sort(roots) + slices.Sort(bundleDirs) t.Logf("%s bundle cleanup: found %d deployment(s) with prefix %q", time.Now().Format(time.RFC3339), len(roots), prefix) @@ -122,6 +124,46 @@ func cleanBundles(ctx context.Context, t *testing.T, execPath, prefix string) { if len(failed) > 0 { t.Logf("WARNING: bundle cleanup failed to destroy %d deployment(s), leaked until swept: %s", len(failed), strings.Join(failed, ", ")) } + + removeBundleDirs(ctx, t, w, bundleDirs, failed) +} + +// removeBundleDirs deletes the ~/.bundle/ directories this run created. +// Destroy empties them but leaves them behind, and the names are never reused, so +// they pile up in the workspace. One whose destroy failed is left for the sweep. +func removeBundleDirs(ctx context.Context, t *testing.T, w *databricks.WorkspaceClient, bundleDirs, failed []string) { + const maxConcurrentDeletes = 20 + sem := make(chan struct{}, maxConcurrentDeletes) + var wg sync.WaitGroup + var mu sync.Mutex + var failedRemovals, kept []string + for _, dir := range bundleDirs { + if slices.ContainsFunc(failed, func(root string) bool { + return root == dir || strings.HasPrefix(root, dir+"/") + }) { + kept = append(kept, dir) + continue + } + sem <- struct{}{} + wg.Go(func() { + defer func() { <-sem }() + err := w.Workspace.Delete(ctx, workspace.Delete{Path: dir, Recursive: true}) + if err != nil && !errors.Is(err, apierr.ErrNotFound) { + mu.Lock() + failedRemovals = append(failedRemovals, dir) + mu.Unlock() + } + }) + } + wg.Wait() + + t.Logf("%s bundle cleanup: removed %d/%d bundle directory(ies), kept %d holding a failed destroy", + time.Now().Format(time.RFC3339), len(bundleDirs)-len(kept)-len(failedRemovals), len(bundleDirs)-len(kept), len(kept)) + + if len(failedRemovals) > 0 { + slices.Sort(failedRemovals) + t.Logf("WARNING: bundle cleanup could not remove %d bundle directory(ies): %s", len(failedRemovals), strings.Join(failedRemovals, ", ")) + } } // findDeploymentRoots walks the workspace tree under dir and returns the paths