bundle: load, include and merge the configuration on structs; delete libs/dyn

.nextchanges/bundles/struct-only-load.md

@@ -0,0 +1 @@
+* Accept `resources: null` in a bundle configuration instead of failing validation. ([#7374](https://github.com/databricks/cli/pull/7374))

AGENTS.md

@@ -73,7 +73,6 @@ GIT_EDITOR=true GIT_SEQUENCE_EDITOR=true VISUAL=true GIT_PAGER=cat git rebase or
 - `bundle/phases/` - High-level deployment phases
 
 **libs/** - Shared libraries and utilities
-- `libs/dyn/` - Dynamic configuration value tree: loads `databricks.yml` and its includes and merges targets, before the configuration is converted to typed structs
 - `libs/structs/structvar/` - Typed configuration values with references and locations
 - `libs/filer/` - File system abstraction (local, DBFS, workspace)
 - `libs/auth/` - Databricks authentication handling

acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/output.txt

@@ -1,9 +1,13 @@
 
 >>> [CLI] bundle validate
+Warning: unknown field: unexpected_after_failure
+  at bundle
+  in b.yml:2:3
+
 Error: cannot merge map with string
 
 Name: test-bundle
 
-Found 1 error
+Found 1 error and 1 warning
 
 Exit code: 1

acceptance/bundle/validate/nulls/resources_container/output.txt

@@ -1,9 +1,9 @@
 
 >>> [CLI] bundle validate
-Error: expected a map at "resources", found nil
-
 Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
 
-Found 1 error
-
-Exit code: 1
+Validation OK!

acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/output.txt

@@ -1,8 +1,13 @@
 
 >>> [CLI] bundle validate
-Error: failed to rewrite [TEST_TMP_DIR]/databricks.yml: expected a map or sequence, found bool
+Warning: expected map, found bool
+  at targets
+  in databricks.yml:4:10
 
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
 
-Found 1 error
-
-Exit code: 1
+Found 1 warning

bundle/config/loader/entry_point.go

@@ -24,7 +24,7 @@ func (m *entryPoint) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics
 	if err != nil {
 		return diag.FromErr(err)
 	}
-	this, diags := config.LoadDyn(path)
+	this, diags := config.Load(path)
 	if diags.HasError() {
 		return diags
 	}

bundle/config/loader/entry_point_test.go

@@ -5,22 +5,21 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/loader"
-	"github.com/databricks/cli/bundle/internal/bundletest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestEntryPointNoRootPath(t *testing.T) {
 	b := &bundle.Bundle{}
-	diags := bundletest.ApplyLoading(t, b, loader.EntryPoint())
+	diags := bundle.Apply(t.Context(), b, loader.EntryPoint())
 	require.Error(t, diags.Error())
 }
 
 func TestEntryPoint(t *testing.T) {
 	b := &bundle.Bundle{
 		BundleRootPath: "testdata/basic",
 	}
-	diags := bundletest.ApplyLoading(t, b, loader.EntryPoint())
+	diags := bundle.Apply(t.Context(), b, loader.EntryPoint())
 	require.NoError(t, diags.Error())
 	assert.Equal(t, "loader_test", b.Config.Bundle.Name)
 }

bundle/config/loader/process_include.go

@@ -4,17 +4,17 @@ import (
 	"cmp"
 	"context"
 	"fmt"
+	"maps"
 	"slices"
 	"strings"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/structs/structpath"
 )
 
-func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics {
+func validateFileFormat(configRoot *config.Root, filePath string) diag.Diagnostics {
 	for _, resourceDescription := range config.SupportedResources() {
 		singularName := resourceDescription.SingularName
 
@@ -29,49 +29,38 @@ func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics
 	return nil
 }
 
-func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.Diagnostics {
+func validateSingleResourceDefined(configRoot *config.Root, ext, typ string) diag.Diagnostics {
 	type resource struct {
-		path  dyn.Path
-		value dyn.Value
-		typ   string
-		key   string
+		path *structpath.PathNode
+		typ  string
+		key  string
 	}
 
 	var resources []resource
 	supportedResources := config.SupportedResources()
 
-	// Gather all resources defined in the resources block.
-	_, err := dyn.MapByPattern(
-		configRoot,
-		dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()),
-		func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
-			// The key for the resource, e.g. "my_job" for jobs.my_job.
-			k := p[2].Key()
-			// The type of the resource, e.g. "job" for jobs.my_job.
-			typ := supportedResources[p[1].Key()].SingularName
-
-			resources = append(resources, resource{path: p, value: v, typ: typ, key: k})
-			return v, nil
+	// Gather all resources defined in the resources block, and in target blocks.
+	configRoot.Resources.IterateAllResources(func(group, name string, _ config.ConfigResource) bool {
+		resources = append(resources, resource{
+			path: structpath.NewPath(nil, "resources", group, name),
+			typ:  supportedResources[group].SingularName,
+			key:  name,
 		})
-	if err != nil {
-		return diag.FromErr(err)
-	}
-
-	// Gather all resources defined in a target block.
-	_, err = dyn.MapByPattern(
-		configRoot,
-		dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey(), dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()),
-		func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
-			// The key for the resource, e.g. "my_job" for jobs.my_job.
-			k := p[4].Key()
-			// The type of the resource, e.g. "job" for jobs.my_job.
-			typ := supportedResources[p[3].Key()].SingularName
-
-			resources = append(resources, resource{path: p, value: v, typ: typ, key: k})
-			return v, nil
+		return true
+	})
+	for _, target := range slices.Sorted(maps.Keys(configRoot.Targets)) {
+		t := configRoot.Targets[target]
+		if t == nil || t.Resources == nil {
+			continue
+		}
+		t.Resources.IterateAllResources(func(group, name string, _ config.ConfigResource) bool {
+			resources = append(resources, resource{
+				path: structpath.NewPath(nil, "targets", target, "resources", group, name),
+				typ:  supportedResources[group].SingularName,
+				key:  name,
+			})
+			return true
 		})
-	if err != nil {
-		return diag.FromErr(err)
 	}
 
 	typeMatch := true
@@ -111,8 +100,8 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D
 	var locations []diag.Location
 	var paths []*structpath.PathNode
 	for _, rr := range resources {
-		locations = append(locations, rr.value.Locations()...)
-		paths = append(paths, dyn.ToStructPath(rr.path))
+		locations = append(locations, configRoot.GetLocationsOfExact(rr.path)...)
+		paths = append(paths, rr.path)
 	}
 	// Sort the locations and paths to make the output deterministic.
 	slices.SortFunc(locations, func(a, b diag.Location) int {
@@ -151,18 +140,32 @@ func (m *processInclude) Name() string {
 }
 
 func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
-	this, diags := config.LoadDyn(m.fullPath)
+	this, diags := m.load()
 	if diags.HasError() {
 		return diags
 	}
 
+	err := b.Config.Merge(this)
+	if err != nil {
+		diags = diags.Extend(diag.FromErr(err))
+	}
+	return diags
+}
+
+// load loads and validates the included file.
+func (m *processInclude) load() (*config.Root, diag.Diagnostics) {
+	this, diags := config.Load(m.fullPath)
+	if diags.HasError() {
+		return nil, diags
+	}
+
 	// Add any diagnostics associated with the file format.
-	diags = append(diags, validateFileFormat(this.Value(), m.relPath)...)
+	diags = append(diags, validateFileFormat(this, m.relPath)...)
 	if diags.HasError() {
-		return diags
+		return nil, diags
 	}
 
-	if include := this.Value().Get("include"); include.Kind() == dyn.KindSequence && len(include.MustSequence()) > 0 {
+	if len(this.Include) > 0 {
 		diags = diags.Append(diag.Diagnostic{
 			Severity: diag.Warning,
 			Summary:  "Include section is defined outside root file",
@@ -173,9 +176,5 @@ Only includes defined in databricks.yml are applied.`,
 		})
 	}
 
-	err := b.Config.Merge(this)
-	if err != nil {
-		diags = diags.Extend(diag.FromErr(err))
-	}
-	return diags
+	return this, diags
 }

bundle/config/loader/process_include_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/loader"
-	"github.com/databricks/cli/bundle/internal/bundletest"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/stretchr/testify/assert"
@@ -31,7 +30,7 @@ func TestProcessInclude(t *testing.T) {
 	assert.Equal(t, "foo", b.Config.Workspace.Host)
 
 	// Apply the mutator and assert that the host value has been updated
-	diags := bundletest.ApplyLoading(t, b, m)
+	diags := bundle.Apply(t.Context(), b, m)
 	require.NoError(t, diags.Error())
 	assert.Equal(t, "bar", b.Config.Workspace.Host)
 }
@@ -55,7 +54,7 @@ func TestProcessIncludeFormatMatch(t *testing.T) {
 			}
 
 			m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName)
-			diags := bundletest.ApplyLoading(t, b, m)
+			diags := bundle.Apply(t.Context(), b, m)
 			assert.Empty(t, diags)
 		})
 	}
@@ -210,7 +209,7 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) {
 			}
 
 			m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName)
-			diags := bundletest.ApplyLoading(t, b, m)
+			diags := bundle.Apply(t.Context(), b, m)
 			require.Len(t, diags, 1)
 			assert.Equal(t, expectedDiags, diags)
 		})

bundle/config/loader/process_root_includes.go

@@ -10,8 +10,6 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
-	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/logdiag"
 )
 
@@ -40,16 +38,11 @@ func hasGlobCharacters(path string) (string, bool) {
 }
 
 func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	var out []bundle.Mutator
-
-	var include []string
-	if err := b.Config.DecodeLoading(&include, "include"); err != nil {
-		return diag.FromErr(err)
-	}
+	var out []*processInclude
 
 	// Preserve the raw include patterns before they are replaced below with the
 	// expanded list of loaded files, so IsFileIncluded can re-match against them.
-	b.SetIncludePatterns(include)
+	b.SetIncludePatterns(b.Config.Include)
 
 	// Map with files we've already seen to avoid loading them twice.
 	seen := map[string]bool{}
@@ -84,7 +77,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 	// For each glob, find all files to load.
 	// Ordering of the list of globs is maintained in the output.
 	// For matches that appear in multiple globs, only the first is kept.
-	for entryIndex, entry := range include {
+	for entryIndex, entry := range b.Config.Include {
 		// Include paths must be relative.
 		if filepath.IsAbs(entry) {
 			return diag.Errorf("%s: includes must be relative paths", entry)
@@ -134,42 +127,40 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
 		slices.Sort(includes)
 		files = append(files, includes...)
 		for _, include := range includes {
-			out = append(out, ProcessInclude(filepath.Join(b.BundleRootPath, include), include))
+			out = append(out, &processInclude{fullPath: filepath.Join(b.BundleRootPath, include), relPath: include})
 		}
 	}
 
 	// Swap out the original includes list with the expanded globs.
-	err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
-		// Include is omitempty in the typed configuration, so an empty list must stay
-		// absent from the dynamic tree rather than be written as [].
-		if len(files) == 0 {
-			return dyn.DropKeys(root, []string{"include"})
-		}
-
-		includeValues := make([]dyn.Value, 0, len(files))
-		for _, file := range files {
-			includeValues = append(includeValues, dyn.V(file))
-		}
-		return dyn.Set(root, "include", dyn.NewValue(includeValues, root.Get("include").Locations()))
-	})
-	if err != nil {
-		return diag.FromErr(err)
+	// Include is omitempty in the typed configuration, so an empty list must stay
+	// absent rather than be written as [].
+	b.Config.Include = nil
+	if len(files) > 0 {
+		b.Config.Include = files
 	}
 
 	// Track number of bundle YAML (or JSON) files in the configuration. The +1 is there
 	// to account for the root databricks.yaml file.
 	b.Metrics.ConfigurationFileCount = int64(len(files)) + 1
 
-	// Like main, the includes are applied without being recorded as mutators.
+	// Load all included files first and merge them into the configuration at once.
+	// Merging them one by one would convert the whole accumulated configuration per
+	// file, making load quadratic in the number of included files.
+	var roots []*config.Root
 	for _, m := range out {
-		mctx := log.NewContext(ctx, log.GetLogger(ctx).With("mutator", m.Name()))
-		log.Debugf(mctx, "Apply")
-		for _, d := range m.Apply(mctx, b) {
-			logdiag.LogDiag(mctx, d)
+		this, diags := m.load()
+		for _, d := range diags {
+			logdiag.LogDiag(ctx, d)
 		}
-		if logdiag.HasError(ctx) {
-			break
+		if diags.HasError() {
+			return nil
 		}
+		roots = append(roots, this)
+	}
+
+	err := b.Config.Merge(roots...)
+	if err != nil {
+		return diag.FromErr(err)
 	}
 	return nil
 }