walk: replace the generated walkers with one driven by cached struct info

bundle/config/merge_parity_test.go

@@ -0,0 +1,125 @@
+package config_test
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/libs/structs/structvar"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// dynMerge merges b into a with the dynamic configuration, like includes do.
+func dynMerge(t *testing.T, fa, fb string, a, b []byte) (string, bool) {
+	ra, diags := config.LoadDynFromBytes(fa, a)
+	if diags.HasError() {
+		return "", false
+	}
+	rb, diags := config.LoadDynFromBytes(fb, b)
+	if diags.HasError() {
+		return "", false
+	}
+	if err := ra.Merge(rb); err != nil {
+		return "merge error: " + err.Error(), true
+	}
+	if _, err := ra.FinishLoading(); err != nil {
+		return "", false
+	}
+	out, _ := json.Marshal(ra)
+	return string(out), true
+}
+
+func typedMerge(t *testing.T, fa, fb string, a, b []byte) string {
+	var ra, rb config.Root
+	sva, _, err := structvar.DecodeYAML(fa, strings.NewReader(string(a)), &ra)
+	if err != nil {
+		return "decode error: " + err.Error()
+	}
+	svb, _, err := structvar.DecodeYAML(fb, strings.NewReader(string(b)), &rb)
+	if err != nil {
+		return "decode error: " + err.Error()
+	}
+	if err := sva.Merge(nil, svb.View()); err != nil {
+		return "merge error: " + err.Error()
+	}
+	out, _ := json.Marshal(&ra)
+	return string(out)
+}
+
+// TestMergeParityWithDyn merges every ordered pair of the YAML files of a directory of
+// the acceptance tests (and of its subdirectories) once with the dynamic configuration,
+// like includes do, and once with structvar.Merge on the typed configuration, and
+// requires the same result. The only difference is the shorthand of a target's variable
+// (`foo: value` for `foo: {default: value}`), which loading rewrites before merging.
+func TestMergeParityWithDyn(t *testing.T) {
+	root := "../../acceptance/bundle"
+	same, diff, skipped, sameButVars := 0, 0, 0, 0
+	byDir := map[string][]string{}
+	err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error {
+		if err == nil && !d.IsDir() && (strings.HasSuffix(p, ".yml") || strings.HasSuffix(p, ".yaml")) {
+			byDir[filepath.Dir(p)] = append(byDir[filepath.Dir(p)], p)
+		}
+		return nil
+	})
+	require.NoError(t, err)
+	// Pairs of the files of a directory (and its subdirectories one level down).
+	var groups [][]string
+	for dir, files := range byDir {
+		g := append([]string(nil), files...)
+		for d2, f2 := range byDir {
+			if filepath.Dir(d2) == dir {
+				g = append(g, f2...)
+			}
+		}
+		groups = append(groups, g)
+	}
+	for _, files := range groups {
+		for _, fa := range files {
+			a, _ := os.ReadFile(fa)
+			for _, fb := range files {
+				b, _ := os.ReadFile(fb)
+				want, ok := dynMerge(t, fa, fb, a, b)
+				if !ok {
+					skipped++
+					continue
+				}
+				got := typedMerge(t, fa, fb, a, b)
+				if got == want {
+					same++
+					continue
+				}
+				if withoutTargetVariables(t, got) == withoutTargetVariables(t, want) {
+					sameButVars++
+					continue
+				}
+				diff++
+				if diff <= 12 {
+					t.Logf("DIFF %s + %s\n dyn:   %.400s\n typed: %.400s", fa, fb, want, got)
+				}
+			}
+		}
+	}
+	assert.Zero(t, diff)
+	t.Logf("same=%d only-target-variables-differ=%d diff=%d skipped=%d", same, sameButVars, diff, skipped)
+}
+
+// withoutTargetVariables drops the variables of the targets (the shorthand rewrite).
+func withoutTargetVariables(t *testing.T, s string) string {
+	var m map[string]any
+	if json.Unmarshal([]byte(s), &m) != nil {
+		return s
+	}
+	if ts, ok := m["targets"].(map[string]any); ok {
+		for _, tv := range ts {
+			if tm, ok := tv.(map[string]any); ok {
+				delete(tm, "variables")
+			}
+		}
+	}
+	out, _ := json.Marshal(m)
+	return string(out)
+}

bundle/config/root.go

@@ -824,6 +824,13 @@ func (r *Root) GetResourceConfig(path string) (any, error) {
 	return typedConfigPtr.Interface(), nil
 }
 
+// IsEmptyAt reports whether the map or slice at path was written as empty ({} or []): a
+// typed value cannot tell it from an unset one. It says nothing about a value that has
+// been filled in since: check that it is still empty.
+func (r Root) IsEmptyAt(path *structpath.PathNode) bool {
+	return r.locations.Sub(path).IsEmpty()
+}
+
 // IsReferenceAt is [Root.IsReference] for a path node.
 func (r Root) IsReferenceAt(path *structpath.PathNode) bool {
 	_, ok := r.refs[path.String()]

bundle/config/root_test.go

@@ -324,3 +324,26 @@ func TestNestedYAMLAnchorCycle(t *testing.T) {
 	_, diags := LoadFromBytes("a.yml", []byte("variables:\n  v:\n    default: &x {a: [*x]}\n"))
 	assert.ErrorContains(t, diags.Error(), `cyclic reference to anchor "x"`)
 }
+
+func TestRootEmpties(t *testing.T) {
+	r, diags := LoadFromBytes("databricks.yml", []byte(`
+resources:
+  jobs:
+    j:
+      name: x
+      permissions: []
+      tags: {}
+    k:
+      name: y
+`))
+	require.False(t, diags.HasError(), diags)
+
+	empty := func(name, field string) bool {
+		return r.IsEmptyAt(structpath.NewPath(nil, "resources", "jobs", name, field))
+	}
+	assert.True(t, empty("j", "permissions"))
+	assert.True(t, empty("j", "tags"))
+	assert.False(t, empty("k", "permissions"))
+	assert.Empty(t, r.Resources.Jobs["j"].Permissions)
+	assert.Empty(t, r.Resources.Jobs["k"].Permissions)
+}

bundle/config/validate/enum.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/walk"
 	"github.com/databricks/cli/libs/diag"
 )
 
@@ -20,7 +21,9 @@ func (f *enum) Name() string {
 // Apply reports enum fields with invalid values. The walker is generated
 // (enum_walk.go) by bundle/internal/validation.
 func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	diags := walkConfig(&b.Config, false, enumConfigRoot)
+	w := &walker{}
+	walk.Enums(&b.Config, w.enumValue)
+	diags := w.diags
 	sortDiagnostics(diags)
 	return diags
 }