diff --git a/.nextchanges/bundles/struct-only-load.md b/.nextchanges/bundles/struct-only-load.md new file mode 100644 index 00000000000..248fa3e98eb --- /dev/null +++ b/.nextchanges/bundles/struct-only-load.md @@ -0,0 +1 @@ +* Accept `resources: null` in a bundle configuration instead of failing validation. ([#7374](https://github.com/databricks/cli/pull/7374)) diff --git a/AGENTS.md b/AGENTS.md index 2ec41d4a3b7..c0cf2704b63 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,7 +73,6 @@ GIT_EDITOR=true GIT_SEQUENCE_EDITOR=true VISUAL=true GIT_PAGER=cat git rebase or - `bundle/phases/` - High-level deployment phases **libs/** - Shared libraries and utilities -- `libs/dyn/` - Dynamic configuration value tree: loads `databricks.yml` and its includes and merges targets, before the configuration is converted to typed structs - `libs/structs/structvar/` - Typed configuration values with references and locations - `libs/filer/` - File system abstraction (local, DBFS, workspace) - `libs/auth/` - Databricks authentication handling diff --git a/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/output.txt b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/output.txt index f5f6443e809..b836ae5fa73 100644 --- a/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/output.txt +++ b/acceptance/bundle/validate/includes/merge_failure_short_circuits_diagnostics/output.txt @@ -1,9 +1,13 @@ >>> [CLI] bundle validate +Warning: unknown field: unexpected_after_failure + at bundle + in b.yml:2:3 + Error: cannot merge map with string Name: test-bundle -Found 1 error +Found 1 error and 1 warning Exit code: 1 diff --git a/acceptance/bundle/validate/nulls/resources_container/output.txt b/acceptance/bundle/validate/nulls/resources_container/output.txt index 1241151fda5..d44a21b582a 100644 --- a/acceptance/bundle/validate/nulls/resources_container/output.txt +++ b/acceptance/bundle/validate/nulls/resources_container/output.txt @@ -1,9 +1,9 @@ >>> [CLI] bundle validate -Error: expected a map at "resources", found nil - Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default -Found 1 error - -Exit code: 1 +Validation OK! diff --git a/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/output.txt b/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/output.txt index 8020d180559..9682b7b460d 100644 --- a/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/output.txt +++ b/acceptance/bundle/validate/targets/scalar_rewrite_diagnostic/output.txt @@ -1,8 +1,13 @@ >>> [CLI] bundle validate -Error: failed to rewrite [TEST_TMP_DIR]/databricks.yml: expected a map or sequence, found bool +Warning: expected map, found bool + at targets + in databricks.yml:4:10 +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default -Found 1 error - -Exit code: 1 +Found 1 warning diff --git a/bundle/config/loader/entry_point.go b/bundle/config/loader/entry_point.go index b44ab81e813..d476cb221c4 100644 --- a/bundle/config/loader/entry_point.go +++ b/bundle/config/loader/entry_point.go @@ -24,7 +24,7 @@ func (m *entryPoint) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics if err != nil { return diag.FromErr(err) } - this, diags := config.LoadDyn(path) + this, diags := config.Load(path) if diags.HasError() { return diags } diff --git a/bundle/config/loader/entry_point_test.go b/bundle/config/loader/entry_point_test.go index cd088fb771c..b61d3f876ac 100644 --- a/bundle/config/loader/entry_point_test.go +++ b/bundle/config/loader/entry_point_test.go @@ -5,14 +5,13 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/loader" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestEntryPointNoRootPath(t *testing.T) { b := &bundle.Bundle{} - diags := bundletest.ApplyLoading(t, b, loader.EntryPoint()) + diags := bundle.Apply(t.Context(), b, loader.EntryPoint()) require.Error(t, diags.Error()) } @@ -20,7 +19,7 @@ func TestEntryPoint(t *testing.T) { b := &bundle.Bundle{ BundleRootPath: "testdata/basic", } - diags := bundletest.ApplyLoading(t, b, loader.EntryPoint()) + diags := bundle.Apply(t.Context(), b, loader.EntryPoint()) require.NoError(t, diags.Error()) assert.Equal(t, "loader_test", b.Config.Bundle.Name) } diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index 2d5c3156ad7..0d4dd6e1d30 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -4,17 +4,17 @@ import ( "cmp" "context" "fmt" + "maps" "slices" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" ) -func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics { +func validateFileFormat(configRoot *config.Root, filePath string) diag.Diagnostics { for _, resourceDescription := range config.SupportedResources() { singularName := resourceDescription.SingularName @@ -29,49 +29,38 @@ func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics return nil } -func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.Diagnostics { +func validateSingleResourceDefined(configRoot *config.Root, ext, typ string) diag.Diagnostics { type resource struct { - path dyn.Path - value dyn.Value - typ string - key string + path *structpath.PathNode + typ string + key string } var resources []resource supportedResources := config.SupportedResources() - // Gather all resources defined in the resources block. - _, err := dyn.MapByPattern( - configRoot, - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // The key for the resource, e.g. "my_job" for jobs.my_job. - k := p[2].Key() - // The type of the resource, e.g. "job" for jobs.my_job. - typ := supportedResources[p[1].Key()].SingularName - - resources = append(resources, resource{path: p, value: v, typ: typ, key: k}) - return v, nil + // Gather all resources defined in the resources block, and in target blocks. + configRoot.Resources.IterateAllResources(func(group, name string, _ config.ConfigResource) bool { + resources = append(resources, resource{ + path: structpath.NewPath(nil, "resources", group, name), + typ: supportedResources[group].SingularName, + key: name, }) - if err != nil { - return diag.FromErr(err) - } - - // Gather all resources defined in a target block. - _, err = dyn.MapByPattern( - configRoot, - dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey(), dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // The key for the resource, e.g. "my_job" for jobs.my_job. - k := p[4].Key() - // The type of the resource, e.g. "job" for jobs.my_job. - typ := supportedResources[p[3].Key()].SingularName - - resources = append(resources, resource{path: p, value: v, typ: typ, key: k}) - return v, nil + return true + }) + for _, target := range slices.Sorted(maps.Keys(configRoot.Targets)) { + t := configRoot.Targets[target] + if t == nil || t.Resources == nil { + continue + } + t.Resources.IterateAllResources(func(group, name string, _ config.ConfigResource) bool { + resources = append(resources, resource{ + path: structpath.NewPath(nil, "targets", target, "resources", group, name), + typ: supportedResources[group].SingularName, + key: name, + }) + return true }) - if err != nil { - return diag.FromErr(err) } typeMatch := true @@ -111,8 +100,8 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D var locations []diag.Location var paths []*structpath.PathNode for _, rr := range resources { - locations = append(locations, rr.value.Locations()...) - paths = append(paths, dyn.ToStructPath(rr.path)) + locations = append(locations, configRoot.GetLocationsOfExact(rr.path)...) + paths = append(paths, rr.path) } // Sort the locations and paths to make the output deterministic. slices.SortFunc(locations, func(a, b diag.Location) int { @@ -151,18 +140,32 @@ func (m *processInclude) Name() string { } func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - this, diags := config.LoadDyn(m.fullPath) + this, diags := m.load() if diags.HasError() { return diags } + err := b.Config.Merge(this) + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + return diags +} + +// load loads and validates the included file. +func (m *processInclude) load() (*config.Root, diag.Diagnostics) { + this, diags := config.Load(m.fullPath) + if diags.HasError() { + return nil, diags + } + // Add any diagnostics associated with the file format. - diags = append(diags, validateFileFormat(this.Value(), m.relPath)...) + diags = append(diags, validateFileFormat(this, m.relPath)...) if diags.HasError() { - return diags + return nil, diags } - if include := this.Value().Get("include"); include.Kind() == dyn.KindSequence && len(include.MustSequence()) > 0 { + if len(this.Include) > 0 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "Include section is defined outside root file", @@ -173,9 +176,5 @@ Only includes defined in databricks.yml are applied.`, }) } - err := b.Config.Merge(this) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) - } - return diags + return this, diags } diff --git a/bundle/config/loader/process_include_test.go b/bundle/config/loader/process_include_test.go index 764d576b157..16ed455e622 100644 --- a/bundle/config/loader/process_include_test.go +++ b/bundle/config/loader/process_include_test.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/loader" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" @@ -31,7 +30,7 @@ func TestProcessInclude(t *testing.T) { assert.Equal(t, "foo", b.Config.Workspace.Host) // Apply the mutator and assert that the host value has been updated - diags := bundletest.ApplyLoading(t, b, m) + diags := bundle.Apply(t.Context(), b, m) require.NoError(t, diags.Error()) assert.Equal(t, "bar", b.Config.Workspace.Host) } @@ -55,7 +54,7 @@ func TestProcessIncludeFormatMatch(t *testing.T) { } m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName) - diags := bundletest.ApplyLoading(t, b, m) + diags := bundle.Apply(t.Context(), b, m) assert.Empty(t, diags) }) } @@ -210,7 +209,7 @@ func TestProcessIncludeFormatNotMatch(t *testing.T) { } m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName) - diags := bundletest.ApplyLoading(t, b, m) + diags := bundle.Apply(t.Context(), b, m) require.Len(t, diags, 1) assert.Equal(t, expectedDiags, diags) }) diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go index 853043eb5e9..36166fa8512 100644 --- a/bundle/config/loader/process_root_includes.go +++ b/bundle/config/loader/process_root_includes.go @@ -10,8 +10,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" ) @@ -40,16 +38,11 @@ func hasGlobCharacters(path string) (string, bool) { } func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var out []bundle.Mutator - - var include []string - if err := b.Config.DecodeLoading(&include, "include"); err != nil { - return diag.FromErr(err) - } + var out []*processInclude // Preserve the raw include patterns before they are replaced below with the // expanded list of loaded files, so IsFileIncluded can re-match against them. - b.SetIncludePatterns(include) + b.SetIncludePatterns(b.Config.Include) // Map with files we've already seen to avoid loading them twice. seen := map[string]bool{} @@ -84,7 +77,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // For each glob, find all files to load. // Ordering of the list of globs is maintained in the output. // For matches that appear in multiple globs, only the first is kept. - for entryIndex, entry := range include { + for entryIndex, entry := range b.Config.Include { // Include paths must be relative. if filepath.IsAbs(entry) { return diag.Errorf("%s: includes must be relative paths", entry) @@ -134,42 +127,40 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. slices.Sort(includes) files = append(files, includes...) for _, include := range includes { - out = append(out, ProcessInclude(filepath.Join(b.BundleRootPath, include), include)) + out = append(out, &processInclude{fullPath: filepath.Join(b.BundleRootPath, include), relPath: include}) } } // Swap out the original includes list with the expanded globs. - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - // Include is omitempty in the typed configuration, so an empty list must stay - // absent from the dynamic tree rather than be written as []. - if len(files) == 0 { - return dyn.DropKeys(root, []string{"include"}) - } - - includeValues := make([]dyn.Value, 0, len(files)) - for _, file := range files { - includeValues = append(includeValues, dyn.V(file)) - } - return dyn.Set(root, "include", dyn.NewValue(includeValues, root.Get("include").Locations())) - }) - if err != nil { - return diag.FromErr(err) + // Include is omitempty in the typed configuration, so an empty list must stay + // absent rather than be written as []. + b.Config.Include = nil + if len(files) > 0 { + b.Config.Include = files } // Track number of bundle YAML (or JSON) files in the configuration. The +1 is there // to account for the root databricks.yaml file. b.Metrics.ConfigurationFileCount = int64(len(files)) + 1 - // Like main, the includes are applied without being recorded as mutators. + // Load all included files first and merge them into the configuration at once. + // Merging them one by one would convert the whole accumulated configuration per + // file, making load quadratic in the number of included files. + var roots []*config.Root for _, m := range out { - mctx := log.NewContext(ctx, log.GetLogger(ctx).With("mutator", m.Name())) - log.Debugf(mctx, "Apply") - for _, d := range m.Apply(mctx, b) { - logdiag.LogDiag(mctx, d) + this, diags := m.load() + for _, d := range diags { + logdiag.LogDiag(ctx, d) } - if logdiag.HasError(ctx) { - break + if diags.HasError() { + return nil } + roots = append(roots, this) + } + + err := b.Config.Merge(roots...) + if err != nil { + return diag.FromErr(err) } return nil } diff --git a/bundle/config/loader/process_root_includes_test.go b/bundle/config/loader/process_root_includes_test.go index 5cceb2a7e8a..9b246d769f6 100644 --- a/bundle/config/loader/process_root_includes_test.go +++ b/bundle/config/loader/process_root_includes_test.go @@ -1,15 +1,16 @@ package loader_test import ( + "path/filepath" "runtime" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/loader" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structvar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -18,7 +19,7 @@ func TestProcessRootIncludesEmpty(t *testing.T) { b := &bundle.Bundle{ BundleRootPath: ".", } - diags := bundletest.ApplyLoading(t, b, loader.ProcessRootIncludes()) + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) } @@ -38,7 +39,7 @@ func TestProcessRootIncludesAbs(t *testing.T) { }, }, } - diags := bundletest.ApplyLoading(t, b, loader.ProcessRootIncludes()) + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) require.True(t, diags.HasError()) assert.ErrorContains(t, diags.Error(), "must be relative paths") } @@ -57,7 +58,7 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) { testutil.Touch(t, b.BundleRootPath, "a.yml") testutil.Touch(t, b.BundleRootPath, "b.yml") - diags := bundletest.ApplyLoading(t, b, loader.ProcessRootIncludes()) + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) assert.Equal(t, []string{"a.yml", "b.yml"}, b.Config.Include) } @@ -76,7 +77,7 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) { testutil.Touch(t, b.BundleRootPath, "a1.yml") testutil.Touch(t, b.BundleRootPath, "b1.yml") - diags := bundletest.ApplyLoading(t, b, loader.ProcessRootIncludes()) + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) assert.Equal(t, []string{"a1.yml", "b1.yml"}, b.Config.Include) } @@ -94,11 +95,34 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) { testutil.Touch(t, b.BundleRootPath, "a.yml") - diags := bundletest.ApplyLoading(t, b, loader.ProcessRootIncludes()) + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) assert.Equal(t, []string{"a.yml"}, b.Config.Include) } +// The expanded include list must be visible in both the typed and the dynamic +// configuration: merging the per-file includes reloads the typed configuration from the +// dynamic tree, so the expanded list must not be overwritten by the raw patterns. +func TestProcessRootIncludesUpdatesDynamicValue(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Include: []string{ + "*.yml", + }, + }, + } + + testutil.Touch(t, b.BundleRootPath, "databricks.yml") + testutil.Touch(t, b.BundleRootPath, "a.yml") + + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) + require.NoError(t, diags.Error()) + assert.Equal(t, []string{"a.yml"}, b.Config.Include) + + assert.Equal(t, []any{"a.yml"}, b.Config.View().Get("include").AsAny()) +} + // An empty include list must stay absent from the dynamic tree: the typed field is // omitempty, so writing [] would add an empty "include" to `bundle validate -o json`. func TestProcessRootIncludesEmptyOmitsDynamicValue(t *testing.T) { @@ -113,10 +137,31 @@ func TestProcessRootIncludesEmptyOmitsDynamicValue(t *testing.T) { testutil.Touch(t, b.BundleRootPath, "databricks.yml") - diags := bundletest.ApplyLoading(t, b, loader.ProcessRootIncludes()) + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) assert.Empty(t, b.Config.Include) - assert.Nil(t, b.Config.Include) + assert.Equal(t, structvar.KindInvalid, b.Config.View().Get("include").Kind()) +} + +// Include sections of included files are not applied, but they are merged into the +// include list after the expanded globs. +func TestProcessRootIncludesKeepsIncludesOfIncludedFiles(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Include: []string{ + "*.yml", + }, + }, + } + + testutil.Touch(t, b.BundleRootPath, "databricks.yml") + testutil.WriteFile(t, filepath.Join(b.BundleRootPath, "a.yml"), "include:\n - c.yml\n") + testutil.Touch(t, b.BundleRootPath, "b.yml") + + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) + require.NoError(t, diags.Error()) + assert.Equal(t, []string{"a.yml", "b.yml", "c.yml"}, b.Config.Include) } func TestProcessRootIncludesNotExists(t *testing.T) { @@ -128,7 +173,7 @@ func TestProcessRootIncludesNotExists(t *testing.T) { }, }, } - diags := bundletest.ApplyLoading(t, b, loader.ProcessRootIncludes()) + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) require.True(t, diags.HasError()) assert.ErrorContains(t, diags.Error(), "notexist.yml defined in 'include' section does not match any files") } @@ -192,7 +237,7 @@ func TestProcessRootIncludesGlobInRootPath(t *testing.T) { BundleRootPath: test.root, } - diags := bundletest.ApplyLoading(t, b, loader.ProcessRootIncludes()) + diags := bundle.Apply(t.Context(), b, loader.ProcessRootIncludes()) require.True(t, diags.HasError()) assert.Len(t, diags, 1) assert.Equal(t, test.diag, diags[0]) diff --git a/bundle/config/merge_parity_test.go b/bundle/config/merge_parity_test.go deleted file mode 100644 index 2c6dd51f563..00000000000 --- a/bundle/config/merge_parity_test.go +++ /dev/null @@ -1,125 +0,0 @@ -package config_test - -import ( - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/structs/structvar" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// dynMerge merges b into a with the dynamic configuration, like includes do. -func dynMerge(t *testing.T, fa, fb string, a, b []byte) (string, bool) { - ra, diags := config.LoadDynFromBytes(fa, a) - if diags.HasError() { - return "", false - } - rb, diags := config.LoadDynFromBytes(fb, b) - if diags.HasError() { - return "", false - } - if err := ra.Merge(rb); err != nil { - return "merge error: " + err.Error(), true - } - if _, err := ra.FinishLoading(); err != nil { - return "", false - } - out, _ := json.Marshal(ra) - return string(out), true -} - -func typedMerge(t *testing.T, fa, fb string, a, b []byte) string { - var ra, rb config.Root - sva, _, err := structvar.DecodeYAML(fa, strings.NewReader(string(a)), &ra) - if err != nil { - return "decode error: " + err.Error() - } - svb, _, err := structvar.DecodeYAML(fb, strings.NewReader(string(b)), &rb) - if err != nil { - return "decode error: " + err.Error() - } - if err := sva.Merge(nil, svb.View()); err != nil { - return "merge error: " + err.Error() - } - out, _ := json.Marshal(&ra) - return string(out) -} - -// TestMergeParityWithDyn merges every ordered pair of the YAML files of a directory of -// the acceptance tests (and of its subdirectories) once with the dynamic configuration, -// like includes do, and once with structvar.Merge on the typed configuration, and -// requires the same result. The only difference is the shorthand of a target's variable -// (`foo: value` for `foo: {default: value}`), which loading rewrites before merging. -func TestMergeParityWithDyn(t *testing.T) { - root := "../../acceptance/bundle" - same, diff, skipped, sameButVars := 0, 0, 0, 0 - byDir := map[string][]string{} - err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { - if err == nil && !d.IsDir() && (strings.HasSuffix(p, ".yml") || strings.HasSuffix(p, ".yaml")) { - byDir[filepath.Dir(p)] = append(byDir[filepath.Dir(p)], p) - } - return nil - }) - require.NoError(t, err) - // Pairs of the files of a directory (and its subdirectories one level down). - var groups [][]string - for dir, files := range byDir { - g := append([]string(nil), files...) - for d2, f2 := range byDir { - if filepath.Dir(d2) == dir { - g = append(g, f2...) - } - } - groups = append(groups, g) - } - for _, files := range groups { - for _, fa := range files { - a, _ := os.ReadFile(fa) - for _, fb := range files { - b, _ := os.ReadFile(fb) - want, ok := dynMerge(t, fa, fb, a, b) - if !ok { - skipped++ - continue - } - got := typedMerge(t, fa, fb, a, b) - if got == want { - same++ - continue - } - if withoutTargetVariables(t, got) == withoutTargetVariables(t, want) { - sameButVars++ - continue - } - diff++ - if diff <= 12 { - t.Logf("DIFF %s + %s\n dyn: %.400s\n typed: %.400s", fa, fb, want, got) - } - } - } - } - assert.Zero(t, diff) - t.Logf("same=%d only-target-variables-differ=%d diff=%d skipped=%d", same, sameButVars, diff, skipped) -} - -// withoutTargetVariables drops the variables of the targets (the shorthand rewrite). -func withoutTargetVariables(t *testing.T, s string) string { - var m map[string]any - if json.Unmarshal([]byte(s), &m) != nil { - return s - } - if ts, ok := m["targets"].(map[string]any); ok { - for _, tv := range ts { - if tm, ok := tv.(map[string]any); ok { - delete(tm, "variables") - } - } - } - out, _ := json.Marshal(m) - return string(out) -} diff --git a/bundle/config/mutator/compute_id_compat.go b/bundle/config/mutator/compute_id_compat.go index 35568441ce8..c1f99becd83 100644 --- a/bundle/config/mutator/compute_id_compat.go +++ b/bundle/config/mutator/compute_id_compat.go @@ -2,11 +2,13 @@ package mutator import ( "context" + "maps" "slices" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type computeIdToClusterId struct{} @@ -22,68 +24,45 @@ func (m *computeIdToClusterId) Name() string { func (m *computeIdToClusterId) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - // The "compute_id" key is set; rewrite it to "cluster_id". - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - v, d := rewriteComputeIdToClusterId(v, dyn.NewPath(dyn.Key("bundle")), nil) - diags = diags.Extend(d) + // Skip if "compute_id" is not set anywhere. + if b.Config.Bundle.ComputeId == "" && !slices.ContainsFunc(slices.Collect(maps.Values(b.Config.Targets)), func(t *config.Target) bool { + return t != nil && t.ComputeId != "" + }) { + return nil + } - // Check if the "compute_id" key is set in any target overrides. - return dyn.MapByPattern(v, dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - v, d := rewriteComputeIdToClusterId(v, dyn.Path{}, p) - diags = diags.Extend(d) - return v, nil - }) - }) + // The "compute_id" key is set; rewrite it to "cluster_id". + b2 := &b.Config.Bundle + diags = diags.Extend(rewriteComputeIdToClusterId(b, structpath.NewStringKey(nil, "bundle"), &b2.ComputeId, &b2.ClusterId)) - diags = diags.Extend(diag.FromErr(err)) + // Check if the "compute_id" key is set in any target overrides. + for _, name := range slices.Sorted(maps.Keys(b.Config.Targets)) { + if t := b.Config.Targets[name]; t != nil { + diags = diags.Extend(rewriteComputeIdToClusterId(b, structpath.NewPath(nil, "targets", name), &t.ComputeId, &t.ClusterId)) + } + } return diags } -// rewriteComputeIdToClusterId rewrites p.compute_id to p.cluster_id in v; prefix is the -// path of v in the configuration, used in the diagnostic. -func rewriteComputeIdToClusterId(v dyn.Value, p, prefix dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - computeIdPath := p.Append(dyn.Key("compute_id")) - computeId, err := dyn.GetByPath(v, computeIdPath) +// rewriteComputeIdToClusterId moves the compute_id of the value at p to its cluster_id, +// with its locations. +func rewriteComputeIdToClusterId(b *bundle.Bundle, p *structpath.PathNode, computeId, clusterId *string) diag.Diagnostics { // If the "compute_id" key is not set, we don't need to do anything. - if err != nil { - return v, nil + if *computeId == "" { + return nil } - if computeId.Kind() == dyn.KindInvalid { - return v, nil - } - - diags = diags.Append(diag.Diagnostic{ + computeIdPath := structpath.NewStringKey(p, "compute_id") + locs := b.Config.GetLocationsOfExact(computeIdPath) + diags := diag.Diagnostics{{ Severity: diag.Warning, Summary: "compute_id is deprecated, please use cluster_id instead", - Locations: computeId.Locations(), - Paths: dyn.ToStructPaths(slices.Concat(prefix, computeIdPath)), - }) + Locations: locs, + Paths: []*structpath.PathNode{computeIdPath}, + }} - clusterIdPath := p.Append(dyn.Key("cluster_id")) - nv, err := dyn.SetByPath(v, clusterIdPath, computeId) - if err != nil { - return dyn.InvalidValue, diag.FromErr(err) - } + *clusterId = *computeId + b.Config.SetLocations(structpath.NewStringKey(p, "cluster_id"), locs) // Drop the "compute_id" key. - vout, err := dyn.Walk(nv, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - switch len(p) { - case 0: - return v, nil - case 1: - if p[0] == dyn.Key("compute_id") { - return v, dyn.ErrDrop - } - return v, nil - case 2: - if p[1] == dyn.Key("compute_id") { - return v, dyn.ErrDrop - } - } - return v, dyn.ErrSkip - }) - - diags = diags.Extend(diag.FromErr(err)) - return vout, diags + return diags.Extend(diag.FromErr(b.Config.Delete(computeIdPath))) } diff --git a/bundle/config/mutator/compute_id_compate_test.go b/bundle/config/mutator/compute_id_compate_test.go index 34cf1df96c7..728bae43191 100644 --- a/bundle/config/mutator/compute_id_compate_test.go +++ b/bundle/config/mutator/compute_id_compate_test.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/diag" "github.com/stretchr/testify/assert" ) @@ -20,7 +19,7 @@ func TestComputeIdToClusterId(t *testing.T) { }, } - diags := bundletest.ApplyLoading(t, b, mutator.ComputeIdToClusterId()) + diags := bundle.Apply(t.Context(), b, mutator.ComputeIdToClusterId()) assert.NoError(t, diags.Error()) assert.Equal(t, "compute-id", b.Config.Bundle.ClusterId) assert.Empty(t, b.Config.Bundle.ComputeId) @@ -41,11 +40,11 @@ func TestComputeIdToClusterIdInTargetOverride(t *testing.T) { }, } - diags := bundletest.ApplyLoading(t, b, mutator.ComputeIdToClusterId()) + diags := bundle.Apply(t.Context(), b, mutator.ComputeIdToClusterId()) assert.NoError(t, diags.Error()) assert.Empty(t, b.Config.Targets["dev"].ComputeId) - diags = diags.Extend(bundletest.ApplyLoading(t, b, mutator.SelectTarget("dev"))) + diags = diags.Extend(bundle.Apply(t.Context(), b, mutator.SelectTarget("dev"))) assert.NoError(t, diags.Error()) assert.Equal(t, "compute-id-dev", b.Config.Bundle.ClusterId) diff --git a/bundle/config/mutator/default_target.go b/bundle/config/mutator/default_target.go index 4ec7b942134..73d99002a06 100644 --- a/bundle/config/mutator/default_target.go +++ b/bundle/config/mutator/default_target.go @@ -5,8 +5,8 @@ import ( "fmt" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" ) type defineDefaultTarget struct { @@ -14,8 +14,7 @@ type defineDefaultTarget struct { } // DefineDefaultTarget adds a target named "default" -// to the configuration if none have been defined. It runs while the configuration -// is loading. +// to the configuration if none have been defined. func DefineDefaultTarget() bundle.Mutator { return &defineDefaultTarget{ name: "default", @@ -27,14 +26,13 @@ func (m *defineDefaultTarget) Name() string { } func (m *defineDefaultTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - targets := v.Get("targets") - if m, ok := targets.AsMap(); ok && m.Len() > 0 { - return v, nil - } - return dyn.Set(v, "targets", dyn.NewValue(map[string]dyn.Value{ - m.name: dyn.V(map[string]dyn.Value{}), - }, targets.Locations())) - }) - return diag.FromErr(err) + // Nothing to do if the configuration has at least 1 target. + if len(b.Config.Targets) > 0 { + return nil + } + + // Define default target. + b.Config.Targets = make(map[string]*config.Target) + b.Config.Targets[m.name] = &config.Target{} + return nil } diff --git a/bundle/config/mutator/default_target_test.go b/bundle/config/mutator/default_target_test.go index eead6b57400..6474b81978d 100644 --- a/bundle/config/mutator/default_target_test.go +++ b/bundle/config/mutator/default_target_test.go @@ -6,14 +6,13 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestDefaultTarget(t *testing.T) { b := &bundle.Bundle{} - diags := bundletest.ApplyLoading(t, b, mutator.DefineDefaultTarget()) + diags := bundle.Apply(t.Context(), b, mutator.DefineDefaultTarget()) require.NoError(t, diags.Error()) env, ok := b.Config.Targets["default"] @@ -29,7 +28,7 @@ func TestDefaultTargetAlreadySpecified(t *testing.T) { }, }, } - diags := bundletest.ApplyLoading(t, b, mutator.DefineDefaultTarget()) + diags := bundle.Apply(t.Context(), b, mutator.DefineDefaultTarget()) require.NoError(t, diags.Error()) _, ok := b.Config.Targets["default"] diff --git a/bundle/config/mutator/environments_compat.go b/bundle/config/mutator/environments_compat.go index 706984ec53c..69862a5c2d2 100644 --- a/bundle/config/mutator/environments_compat.go +++ b/bundle/config/mutator/environments_compat.go @@ -2,11 +2,10 @@ package mutator import ( "context" - "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type environmentsToTargets struct{} @@ -22,45 +21,25 @@ func (m *environmentsToTargets) Name() string { func (m *environmentsToTargets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Short circuit if the "environments" key is not set. // This is the common case. - if k := b.Config.Value().Get("environments").Kind(); k == dyn.KindInvalid || k == dyn.KindNil { + if b.Config.Environments == nil { return nil } // The "environments" key is set; validate and rewrite it to "targets". - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - environments := v.Get("environments") - targets := v.Get("targets") - - // Return an error if both "environments" and "targets" are set. - if environments.Kind() != dyn.KindInvalid && targets.Kind() != dyn.KindInvalid { - return dyn.InvalidValue, fmt.Errorf( - "both 'environments' and 'targets' are specified; only 'targets' should be used: %s", - environments.Location().String(), - ) - } - - // Rewrite "environments" to "targets". - if environments.Kind() != dyn.KindInvalid && targets.Kind() == dyn.KindInvalid { - nv, err := dyn.Set(v, "targets", environments) - if err != nil { - return dyn.InvalidValue, err - } - // Drop the "environments" key. - return dyn.Walk(nv, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - switch len(p) { - case 0: - return v, nil - case 1: - if p[0] == dyn.Key("environments") { - return v, dyn.ErrDrop - } - } - return v, dyn.ErrSkip - }) - } - - return v, nil - }) + environmentsPath := structpath.NewStringKey(nil, "environments") + targetsPath := structpath.NewStringKey(nil, "targets") + + // Return an error if both "environments" and "targets" are set. + if b.Config.Targets != nil || b.Config.IsReferenceAt(targetsPath) { + return diag.Errorf( + "both 'environments' and 'targets' are specified; only 'targets' should be used: %s", + b.Config.GetLocationOfExact(environmentsPath).String(), + ) + } - return diag.FromErr(err) + // Rewrite "environments" to "targets" and drop the "environments" key. + b.Config.Targets = b.Config.Environments + b.Config.Environments = nil + b.Config.MoveMetadata(environmentsPath, targetsPath) + return nil } diff --git a/bundle/config/mutator/environments_compat_test.go b/bundle/config/mutator/environments_compat_test.go index 987e41b8ab4..0e7d4caa079 100644 --- a/bundle/config/mutator/environments_compat_test.go +++ b/bundle/config/mutator/environments_compat_test.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -27,7 +26,7 @@ func TestEnvironmentsToTargetsWithBothDefined(t *testing.T) { }, } - diags := bundletest.ApplyLoading(t, b, mutator.EnvironmentsToTargets()) + diags := bundle.Apply(t.Context(), b, mutator.EnvironmentsToTargets()) assert.ErrorContains(t, diags.Error(), `both 'environments' and 'targets' are specified;`) } @@ -42,7 +41,7 @@ func TestEnvironmentsToTargetsWithEnvironmentsDefined(t *testing.T) { }, } - diags := bundletest.ApplyLoading(t, b, mutator.EnvironmentsToTargets()) + diags := bundle.Apply(t.Context(), b, mutator.EnvironmentsToTargets()) require.NoError(t, diags.Error()) assert.Empty(t, b.Config.Environments) assert.Len(t, b.Config.Targets, 1) @@ -59,7 +58,7 @@ func TestEnvironmentsToTargetsWithTargetsDefined(t *testing.T) { }, } - diags := bundletest.ApplyLoading(t, b, mutator.EnvironmentsToTargets()) + diags := bundle.Apply(t.Context(), b, mutator.EnvironmentsToTargets()) require.NoError(t, diags.Error()) assert.Empty(t, b.Config.Environments) assert.Len(t, b.Config.Targets, 1) diff --git a/bundle/config/mutator/initialize_variables.go b/bundle/config/mutator/initialize_variables.go index 70d0d55ac62..e72cdde3103 100644 --- a/bundle/config/mutator/initialize_variables.go +++ b/bundle/config/mutator/initialize_variables.go @@ -4,15 +4,13 @@ import ( "context" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" ) type initializeVariables struct{} // InitializeVariables initializes nil variables to their corresponding zero values. -// It runs while the configuration is loading, so that target overrides can set the -// fields of a variable that is declared without any. func InitializeVariables() bundle.Mutator { return &initializeVariables{} } @@ -22,16 +20,12 @@ func (m *initializeVariables) Name() string { } func (m *initializeVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - if v.Get("variables").Kind() != dyn.KindMap { - return v, nil + vars := b.Config.Variables + for k, v := range vars { + if v == nil { + vars[k] = &variable.Variable{} } - return dyn.Map(v, "variables", dyn.Foreach(func(_ dyn.Path, variable dyn.Value) (dyn.Value, error) { - if variable.Kind() != dyn.KindNil { - return variable, nil - } - return dyn.NewValue(map[string]dyn.Value{}, variable.Locations()), nil - })) - }) - return diag.FromErr(err) + } + + return nil } diff --git a/bundle/config/mutator/initialize_variables_test.go b/bundle/config/mutator/initialize_variables_test.go index d3ceb4c135a..53899baf617 100644 --- a/bundle/config/mutator/initialize_variables_test.go +++ b/bundle/config/mutator/initialize_variables_test.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/variable" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -23,7 +22,7 @@ func TestInitializeVariables(t *testing.T) { }, }, } - diags := bundletest.ApplyLoading(t, b, mutator.InitializeVariables()) + diags := bundle.Apply(t.Context(), b, mutator.InitializeVariables()) require.NoError(t, diags.Error()) assert.NotNil(t, b.Config.Variables["foo"]) assert.NotNil(t, b.Config.Variables["bar"]) @@ -36,7 +35,7 @@ func TestInitializeVariablesWithoutVariables(t *testing.T) { Variables: nil, }, } - diags := bundletest.ApplyLoading(t, b, mutator.InitializeVariables()) + diags := bundle.Apply(t.Context(), b, mutator.InitializeVariables()) require.NoError(t, diags.Error()) assert.Nil(t, b.Config.Variables) } diff --git a/bundle/config/mutator/mutator.go b/bundle/config/mutator/mutator.go index 9fe4fc1346f..a00488a50fa 100644 --- a/bundle/config/mutator/mutator.go +++ b/bundle/config/mutator/mutator.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle/config/loader" "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/bundle/scripts" - "github.com/databricks/cli/libs/logdiag" ) func DefaultMutators(ctx context.Context, b *bundle.Bundle) { @@ -35,17 +34,3 @@ func DefaultMutators(ctx context.Context, b *bundle.Bundle) { validate.UniqueResourceKeys(), ) } - -// FinishLoading converts the configuration of b, loaded and merged by -// [DefaultMutators] (and a target selection), into the typed configuration. It is not -// a mutator: it runs once, at the end of the load phase, also if loading failed so -// that the configuration can still be inspected. -func FinishLoading(ctx context.Context, b *bundle.Bundle) { - diags, err := b.Config.FinishLoading() - for _, d := range diags { - logdiag.LogDiag(ctx, d) - } - if err != nil { - logdiag.LogError(ctx, err) - } -} diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index eaaaa1a64cd..dc7d2f9c862 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -463,12 +463,9 @@ resources: sp: {run_as: {service_principal_name: other_sp}} group: {run_as: {group_name: other_group}} `, tc.root, tc.target) - r, diags := config.LoadDynFromBytes("databricks.yml", []byte(yaml)) + r, diags := config.LoadFromBytes("databricks.yml", []byte(yaml)) require.NoError(t, diags.Error()) require.NoError(t, r.MergeTargetOverrides("test")) - diags, err := r.FinishLoading() - require.NoError(t, err) - require.Empty(t, diags) b := &bundle.Bundle{Config: *r} diags = bundle.Apply(t.Context(), b, SetRunAs()) require.NoError(t, diags.Error()) diff --git a/bundle/config/mutator/select_default_target.go b/bundle/config/mutator/select_default_target.go index d84a0b62180..ad8132a46aa 100644 --- a/bundle/config/mutator/select_default_target.go +++ b/bundle/config/mutator/select_default_target.go @@ -2,6 +2,8 @@ package mutator import ( "context" + "maps" + "slices" "strings" "github.com/databricks/cli/bundle" @@ -20,13 +22,12 @@ func (m *selectDefaultTarget) Name() string { } func (m *selectDefaultTarget) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - // This runs while the configuration is loading. - names := targetNames(b) - if len(names) == 0 { + if len(b.Config.Targets) == 0 { return diag.Errorf("no targets defined") } // One target means there's only one default. + names := slices.Collect(maps.Keys(b.Config.Targets)) if len(names) == 1 { bundle.ApplyContext(ctx, b, SelectTarget(names[0])) return nil @@ -34,12 +35,8 @@ func (m *selectDefaultTarget) Apply(ctx context.Context, b *bundle.Bundle) diag. // Multiple targets means we look for the `default` flag. var defaults []string - for _, name := range names { - var isDefault bool - if err := b.Config.DecodeLoading(&isDefault, "targets", name, "default"); err != nil { - return diag.FromErr(err) - } - if isDefault { + for name, env := range b.Config.Targets { + if env != nil && env.Default { defaults = append(defaults, name) } } diff --git a/bundle/config/mutator/select_default_target_test.go b/bundle/config/mutator/select_default_target_test.go index 0f9fe1e74d1..850bfd88546 100644 --- a/bundle/config/mutator/select_default_target_test.go +++ b/bundle/config/mutator/select_default_target_test.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/stretchr/testify/assert" ) @@ -16,7 +15,7 @@ func TestSelectDefaultTargetNoTargets(t *testing.T) { Targets: map[string]*config.Target{}, }, } - diags := bundletest.ApplyLoading(t, b, mutator.SelectDefaultTarget()) + diags := bundle.Apply(t.Context(), b, mutator.SelectDefaultTarget()) assert.ErrorContains(t, diags.Error(), "no targets defined") } @@ -28,7 +27,7 @@ func TestSelectDefaultTargetSingleTargets(t *testing.T) { }, }, } - diags := bundletest.ApplyLoading(t, b, mutator.SelectDefaultTarget()) + diags := bundle.Apply(t.Context(), b, mutator.SelectDefaultTarget()) assert.NoError(t, diags.Error()) assert.Equal(t, "foo", b.Config.Bundle.Target) } @@ -43,7 +42,7 @@ func TestSelectDefaultTargetNoDefaults(t *testing.T) { }, }, } - diags := bundletest.ApplyLoading(t, b, mutator.SelectDefaultTarget()) + diags := bundle.Apply(t.Context(), b, mutator.SelectDefaultTarget()) assert.ErrorContains(t, diags.Error(), "please specify target") } @@ -56,7 +55,7 @@ func TestSelectDefaultTargetNoDefaultsWithNil(t *testing.T) { }, }, } - diags := bundletest.ApplyLoading(t, b, mutator.SelectDefaultTarget()) + diags := bundle.Apply(t.Context(), b, mutator.SelectDefaultTarget()) assert.ErrorContains(t, diags.Error(), "please specify target") } @@ -70,7 +69,7 @@ func TestSelectDefaultTargetMultipleDefaults(t *testing.T) { }, }, } - diags := bundletest.ApplyLoading(t, b, mutator.SelectDefaultTarget()) + diags := bundle.Apply(t.Context(), b, mutator.SelectDefaultTarget()) assert.ErrorContains(t, diags.Error(), "multiple targets are marked as default") } @@ -84,7 +83,7 @@ func TestSelectDefaultTargetSingleDefault(t *testing.T) { }, }, } - diags := bundletest.ApplyLoading(t, b, mutator.SelectDefaultTarget()) + diags := bundle.Apply(t.Context(), b, mutator.SelectDefaultTarget()) assert.NoError(t, diags.Error()) assert.Equal(t, "bar", b.Config.Bundle.Target) } diff --git a/bundle/config/mutator/select_target.go b/bundle/config/mutator/select_target.go index 831214c2d27..834255463db 100644 --- a/bundle/config/mutator/select_target.go +++ b/bundle/config/mutator/select_target.go @@ -3,12 +3,13 @@ package mutator import ( "context" "fmt" + "maps" + "slices" "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" ) type selectTarget struct { @@ -16,7 +17,7 @@ type selectTarget struct { } // SelectTarget merges the specified target into the root configuration. -// It runs while the configuration is loading. +// After merging, it removes the 'Targets' section from the configuration. func SelectTarget(name string) bundle.Mutator { return &selectTarget{ name: name, @@ -27,33 +28,15 @@ func (m *selectTarget) Name() string { return fmt.Sprintf("SelectTarget(%s)", m.name) } -// targetNames returns the names of the targets of the loading configuration -// (nil if there are none). -func targetNames(b *bundle.Bundle) []string { - targets, ok := b.Config.Value().Get("targets").AsMap() - if !ok { - return nil - } - names := make([]string, 0, targets.Len()) - for _, p := range targets.Pairs() { - names = append(names, p.Key.MustString()) - } - return names -} - func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - if k := b.Config.Value().Get("targets").Kind(); k == dyn.KindInvalid || k == dyn.KindNil { + if b.Config.Targets == nil { return diag.Errorf("no targets defined") } // Get specified target - if !b.Config.HasTarget(m.name) { - return diag.Errorf("%s: no such target. Available targets: %s", m.name, strings.Join(targetNames(b), ", ")) - } - - var target *config.Target - if err := b.Config.DecodeLoading(&target, "targets", m.name); err != nil { - return diag.FromErr(err) + target, ok := b.Config.Targets[m.name] + if !ok { + return diag.Errorf("%s: no such target. Available targets: %s", m.name, strings.Join(slices.Collect(maps.Keys(b.Config.Targets)), ", ")) } // Merge specified target into root configuration structure. @@ -64,23 +47,25 @@ func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti // Store specified target in configuration for reference. b.Target = target - b.Metrics.TargetCount = int64(len(targetNames(b))) + b.Config.Bundle.Target = m.name - err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - v, err := dyn.SetByPath(v, dyn.NewPath(dyn.Key("bundle"), dyn.Key("target")), dyn.V(m.name)) - if err != nil { - return v, err - } + // We do this for backward compatibility. + // TODO: remove when Environments section is not supported anymore. + b.Config.Bundle.Environment = b.Config.Bundle.Target - // We need to keep environment in sync for backwards compatibility. - v, err = dyn.SetByPath(v, dyn.NewPath(dyn.Key("bundle"), dyn.Key("environment")), dyn.V(m.name)) - if err != nil { - return v, err + // Record number of targets in the bundle. This is then used downstream during + // telemetry upload. This value is always >= 1 since if no targets are defined + // in YAML, we create a "default" placeholder target upstream. + b.Metrics.TargetCount = int64(len(b.Config.Targets)) + + // Cleanup the original targets and environments sections since they + // show up in the JSON output of the 'summary' and 'validate' commands. + // Delete (not just clear) them so their locations go too. + for _, k := range []string{"targets", "environments"} { + if err := b.Config.Delete(structpath.NewPath(nil, k)); err != nil { + return diag.FromErr(err) } + } - // Cleanup the original targets and environments sections since they - // show up in the JSON output of the 'summary' and 'validate' commands. - return dyn.DropKeys(v, []string{"targets", "environments"}) - }) - return diag.FromErr(err) + return nil } diff --git a/bundle/config/mutator/select_target_test.go b/bundle/config/mutator/select_target_test.go index e3de623163c..14cec05d9b7 100644 --- a/bundle/config/mutator/select_target_test.go +++ b/bundle/config/mutator/select_target_test.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -26,7 +25,7 @@ func TestSelectTarget(t *testing.T) { }, }, } - diags := bundletest.ApplyLoading(t, b, mutator.SelectTarget("default")) + diags := bundle.Apply(t.Context(), b, mutator.SelectTarget("default")) require.NoError(t, diags.Error()) assert.Equal(t, "bar", b.Config.Workspace.Host) } @@ -39,6 +38,6 @@ func TestSelectTargetNotFound(t *testing.T) { }, }, } - diags := bundletest.ApplyLoading(t, b, mutator.SelectTarget("doesnt-exist")) + diags := bundle.Apply(t.Context(), b, mutator.SelectTarget("doesnt-exist")) require.Error(t, diags.Error(), "no targets defined") } diff --git a/bundle/config/mutator/verify_cli_version.go b/bundle/config/mutator/verify_cli_version.go index e04a2762337..a6d705a6fbb 100644 --- a/bundle/config/mutator/verify_cli_version.go +++ b/bundle/config/mutator/verify_cli_version.go @@ -18,17 +18,12 @@ func VerifyCliVersion() bundle.Mutator { type verifyCliVersion struct{} func (v *verifyCliVersion) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - // This runs while the configuration is loading. - var constraint string - if err := b.Config.DecodeLoading(&constraint, "bundle", "databricks_cli_version"); err != nil { - return diag.FromErr(err) - } - // No constraints specified, skip the check. - if constraint == "" { + if b.Config.Bundle.DatabricksCliVersion == "" { return nil } + constraint := b.Config.Bundle.DatabricksCliVersion if err := validateConstraintSyntax(constraint); err != nil { return diag.FromErr(err) } diff --git a/bundle/config/mutator/verify_cli_version_test.go b/bundle/config/mutator/verify_cli_version_test.go index b3de0a20d08..b9f896a72bb 100644 --- a/bundle/config/mutator/verify_cli_version_test.go +++ b/bundle/config/mutator/verify_cli_version_test.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/internal/build" "github.com/stretchr/testify/require" ) @@ -132,7 +131,7 @@ func TestVerifyCliVersion(t *testing.T) { }, }, } - diags := bundletest.ApplyLoading(t, b, VerifyCliVersion()) + diags := bundle.Apply(t.Context(), b, VerifyCliVersion()) if tc.expectedError != "" { require.NotEmpty(t, diags) require.Contains(t, diags[0].Summary, tc.expectedError) diff --git a/bundle/config/root.go b/bundle/config/root.go index 1dbc24c86c9..ae8da9e7da9 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -15,13 +15,10 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/merge" - "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/service/jobs" + "go.yaml.in/yaml/v3" ) type Script struct { @@ -37,10 +34,6 @@ type Script struct { } type Root struct { //nolint:recvcheck // value receivers for read-only accessors, pointer for mutators - // value holds the configuration tree while the configuration is loading - // (see [Root.Loading]); the fields below are zero until then. - value dyn.Value - // refs holds the pure variable references in fields that cannot hold a string // (e.g. `max_retries: ${var.n}`, where the typed field is zero), keyed by path. refs map[string]string @@ -109,8 +102,19 @@ type Root struct { //nolint:recvcheck // value receivers for read-only accessors Definitions any `json:"definitions,omitempty" bundle:"internal"` } -// Load loads the bundle configuration file at the specified path into a typed -// configuration (see [LoadDyn] to load it for merging). +// loadError returns the diagnostics for the error of loading the file at path. +func loadError(path string, diags diag.Diagnostics, err error) diag.Diagnostics { + if le, ok := errors.AsType[*structvar.LocationError](err); ok { + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: le.Summary, + Locations: []diag.Location{le.Loc}, + }} + } + return diags.Extend(diag.Errorf("failed to load %s: %v", path, err)) +} + +// Load loads the bundle configuration file at the specified path. func Load(path string) (*Root, diag.Diagnostics) { raw, err := os.ReadFile(path) if err != nil { @@ -120,61 +124,27 @@ func Load(path string) (*Root, diag.Diagnostics) { return LoadFromBytes(path, raw) } -// LoadFromBytes is [Load] for the contents of a file. func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { - r, diags := LoadDynFromBytes(path, raw) - if diags.HasError() { - return nil, diags - } - finishDiags, err := r.FinishLoading() - diags = diags.Extend(finishDiags) + node, err := structvar.ParseYAML(bytes.NewReader(raw)) if err != nil { - return nil, diags.Extend(diag.Errorf("failed to load %s: %v", path, err)) + return nil, diag.Errorf("failed to load %s: %v", path, err) } - return r, diags -} -// LoadDyn loads the bundle configuration file at the specified path into a -// configuration that is loading (see [Root.Loading]), to be merged with others. -func LoadDyn(path string) (*Root, diag.Diagnostics) { - raw, err := os.ReadFile(path) - if err != nil { - return nil, diag.FromErr(err) - } - - return LoadDynFromBytes(path, raw) -} - -// LoadDynFromBytes is [LoadDyn] for the contents of a file. -func LoadDynFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { - // Load configuration tree from YAML. - v, err := yamlloader.LoadYAML(path, bytes.NewBuffer(raw)) - if err != nil { - if le, ok := errors.AsType[*yamlloader.LocationError](err); ok { - return nil, diag.Diagnostics{{ - Severity: diag.Error, - Summary: le.Summary, - Locations: []diag.Location{le.Loc}, - }} - } - return nil, diag.Errorf("failed to load %s: %v", path, err) + // Report the errors of the YAML first: the rewrite below follows merge keys and aliases. + if err := structvar.ValidateYAMLNode(path, node); err != nil { + return nil, loadError(path, nil, err) } // Rewrite configuration tree where necessary. - v, err = rewriteShorthands(v) - if err != nil { - return nil, diag.Errorf("failed to rewrite %s: %v", path, err) - } + node = rewriteShorthands(node) - // Normalize dynamic configuration tree according to configuration type. - nv, diags := convert.Normalize(Root{}, v) - if !nv.IsValid() { - // The document is not a map: report the error main's conversion to the - // typed configuration reported. - err := convert.ToTyped(&Root{}, v) - return nil, diags.Extend(diag.Errorf("failed to load %s: %v", path, err)) + var r Root + sv, diags, err := structvar.DecodeYAMLNode(path, node, &r, nil) + if err != nil { + return nil, loadError(path, diags, err) } - return &Root{value: nv}, diags + r.store(sv) + return &r, diags } // LoadFromReader decodes the configuration in r, recording the locations mapLocations @@ -193,66 +163,6 @@ func LoadFromReader(path string, r io.Reader, mapLocations structvar.LocationMap return &root, diags, nil } -// Loading reports whether the configuration is loading: it is a configuration tree -// that the load phase merges (includes, target overrides) and only [Root.Value], -// [Root.Mutate], [Root.Merge] and [Root.MergeTargetOverrides] may be used. The typed -// fields are zero until [Root.FinishLoading] converts the tree into them. -func (r *Root) Loading() bool { - return r.value.IsValid() -} - -// Value returns the configuration tree of a loading configuration -// (invalid if it is not loading). -func (r *Root) Value() dyn.Value { - return r.value -} - -// StartLoading makes the configuration loading, converting the typed configuration -// into a configuration tree if it is not (like main did on entry of the first -// mutator). A bundle's configuration starts out empty and the entry point merges the -// loaded configuration into it; tests may start with a typed configuration. -func (r *Root) StartLoading() error { - if r.Loading() { - return nil - } - v, err := convert.FromTyped(r, dyn.NilValue) - if err != nil { - return err - } - *r = Root{value: v} - return nil -} - -// Mutate applies fn to the configuration tree of a loading configuration. -func (r *Root) Mutate(fn func(dyn.Value) (dyn.Value, error)) error { - if err := r.StartLoading(); err != nil { - return err - } - nv, err := fn(r.value) - if err != nil { - return err - } - r.value = nv - return nil -} - -// FinishLoading converts the configuration tree of a loading configuration into the -// typed configuration, its references and locations, once. The diagnostics explain -// values that could not be converted and were dropped. -func (r *Root) FinishLoading() (diag.Diagnostics, error) { - if !r.Loading() { - return nil, nil - } - var out Root - sv, diags, err := structvar.DecodeDyn(r.value, &out) - if err != nil { - return diags, err - } - *r = out - r.store(sv) - return diags, nil -} - // vars returns the configuration as a [structvar.StructVar]; use [Root.store] to // keep changes made to its references and locations. func (r *Root) vars() *structvar.StructVar { @@ -292,74 +202,34 @@ func (r *Root) InitializeVariables(vars []string) error { return nil } -// Merge merges the other loading configurations into this one, in order. +// Merge merges the other configurations into this one, in order. func (r *Root) Merge(others ...*Root) error { - if err := r.StartLoading(); err != nil { - return err - } for _, other := range others { - nv, err := merge.Merge(r.value, other.value) - if err != nil { + if err := r.MergeAt(nil, other.View()); err != nil { return err } - r.value = nv } return nil } -// HasTarget reports whether the loading configuration defines the target name. -func (r *Root) HasTarget(name string) bool { - _, err := dyn.GetByPath(r.value, dyn.NewPath(dyn.Key("targets"), dyn.Key(name))) - return err == nil -} - -func mergeField(rv, ov dyn.Value, name string) (dyn.Value, error) { - path := dyn.NewPath(dyn.Key(name)) - reference, _ := dyn.GetByPath(rv, path) - override, _ := dyn.GetByPath(ov, path) - - // Merge the override into the reference. - var out dyn.Value - var err error - if reference.IsValid() && override.IsValid() { - out, err = merge.Merge(reference, override) - if err != nil { - return dyn.InvalidValue, err - } - } else if reference.IsValid() { - out = reference - } else if override.IsValid() { - out = override - } else { - return rv, nil - } - - return dyn.SetByPath(rv, path, out) -} +var bundleGitPath = structpath.NewPath(nil, "bundle", "git") func (r *Root) MergeTargetOverrides(name string) error { - root := r.value - target, err := dyn.GetByPath(root, dyn.NewPath(dyn.Key("targets"), dyn.Key(name))) - if err != nil { - return err - } - - // Main converted the configuration to the typed configuration and back around - // every mutator, so the target only held what its typed form holds (e.g. not an - // explicitly null run_as, which would otherwise replace the root's run_as below). - target, err = roundTripTarget(target) - if err != nil { - return err + targetPath := structpath.NewPath(nil, "targets", name) + target := r.View().Lookup(targetPath) + if !target.IsValid() { + return fmt.Errorf("target %s not found", name) } // Confirm validity of variable overrides. - err = validateVariableOverrides(root, target) + err := validateVariableOverrides(r.Variables, r.Targets[name]) if err != nil { return err } - // Merge fields that can be merged 1:1. - for _, f := range []string{ + // Merge fields that can be merged 1:1. Check all of them first so that a failed + // merge leaves the configuration unchanged. + fields := []string{ "bundle", "workspace", "artifacts", @@ -367,115 +237,73 @@ func (r *Root) MergeTargetOverrides(name string) error { "sync", "permissions", "presets", - } { - if root, err = mergeField(root, target, f); err != nil { + } + for _, f := range fields { + if err := structvar.CheckMerge(r.View().Get(f), target.Get(f)); err != nil { + return fmt.Errorf("failed to merge target=%s field=%s: %w", name, f, err) + } + } + if err := structvar.CheckMerge(r.View().Lookup(bundleGitPath), target.Get("git")); err != nil { + return err + } + for _, f := range fields { + if err := r.MergeAt(structpath.NewStringKey(nil, f), target.Get(f)); err != nil { return fmt.Errorf("failed to merge target=%s field=%s: %w", name, f, err) } } // Merge `variables`. This field must be overwritten if set, not merged. - if v := target.Get("variables"); v.Kind() != dyn.KindInvalid { - _, err = dyn.Map(v, ".", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { - varPath := dyn.MustPathFromString("variables").Append(p...) - - vDefault := variable.Get("default") - if vDefault.Kind() != dyn.KindInvalid { - defaultPath := varPath.Append(dyn.Key("default")) - root, err = dyn.SetByPath(root, defaultPath, vDefault) - if err != nil { - return root, err - } + for varName, variable := range target.Get("variables").MapItems() { + varPath := structpath.NewPath(nil, "variables", varName) - // If the target explicitly sets a default value, drop any lookup from the - // root variable definition so SetVariables can assign this default. - lookupPath := varPath.Append(dyn.Key("lookup")) - root, err = dyn.SetByPath(root, lookupPath, dyn.NilValue) + if vDefault := variable.Get("default"); vDefault.IsValid() { + if err := r.Assign(structpath.NewStringKey(varPath, "default"), vDefault); err != nil { + return err } - vLookup := variable.Get("lookup") - if vLookup.Kind() != dyn.KindInvalid { - lookupPath := varPath.Append(dyn.Key("lookup")) - root, err = dyn.SetByPath(root, lookupPath, vLookup) - if err != nil { - return root, err - } + // If the target explicitly sets a default value, drop any lookup from the + // root variable definition so SetVariables can assign this default. + if err := r.Delete(structpath.NewStringKey(varPath, "lookup")); err != nil { + return err + } + } - // If the target explicitly sets a lookup, drop any default value from the - // root variable definition so lookup resolution remains authoritative. - defaultPath := varPath.Append(dyn.Key("default")) - root, err = dyn.SetByPath(root, defaultPath, dyn.NilValue) + if vLookup := variable.Get("lookup"); vLookup.IsValid() { + if err := r.Assign(structpath.NewStringKey(varPath, "lookup"), vLookup); err != nil { + return err } - return root, err - })) - if err != nil { - return err + // If the target explicitly sets a lookup, drop any default value from the + // root variable definition so lookup resolution remains authoritative. + if err := r.Delete(structpath.NewStringKey(varPath, "default")); err != nil { + return err + } } } // Merge `run_as`. This field must be overwritten if set, not merged. - if v := target.Get("run_as"); v.Kind() != dyn.KindInvalid { - root, err = dyn.Set(root, "run_as", v) - if err != nil { - return err - } - } - - // Below, we're setting fields on the bundle key, so make sure it exists. - if root.Get("bundle").Kind() == dyn.KindInvalid { - root, err = dyn.Set(root, "bundle", dyn.V(map[string]dyn.Value{})) - if err != nil { + if v := target.Get("run_as"); v.IsValid() { + if err := r.Assign(structpath.NewStringKey(nil, "run_as"), v); err != nil { return err } } // Merge `mode`. This field must be overwritten if set, not merged. - if v := target.Get("mode"); v.Kind() != dyn.KindInvalid { - root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("mode")), v) - if err != nil { + if v := target.Get("mode"); v.IsValid() { + if err := r.Assign(structpath.NewPath(nil, "bundle", "mode"), v); err != nil { return err } } // Merge `cluster_id`. This field must be overwritten if set, not merged. - if v := target.Get("cluster_id"); v.Kind() != dyn.KindInvalid { - root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("cluster_id")), v) - if err != nil { + if v := target.Get("cluster_id"); v.IsValid() { + if err := r.Assign(structpath.NewPath(nil, "bundle", "cluster_id"), v); err != nil { return err } } // Merge `git`. - if v := target.Get("git"); v.Kind() != dyn.KindInvalid { - ref, err := dyn.GetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("git"))) - if err != nil { - ref = dyn.V(map[string]dyn.Value{}) - } - - // Merge the override into the reference. - out, err := merge.Merge(ref, v) - if err != nil { - return err - } - - // Set the merged value. - root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("git")), out) - if err != nil { - return err - } - } - - r.value = root - return nil -} - -// roundTripTarget converts the target to the typed [Target] and back. -func roundTripTarget(v dyn.Value) (dyn.Value, error) { - var t *Target - if err := convert.ToTyped(&t, v); err != nil { - return dyn.InvalidValue, err - } - return convert.FromTyped(t, v) + return r.MergeAt(bundleGitPath, target.Get("git")) } var allowedVariableDefinitions = []([]string){ @@ -487,30 +315,25 @@ var allowedVariableDefinitions = []([]string){ {"lookup"}, } -// isFullVariableOverrideDef checks if the given value is a full syntax variable override. -// A full syntax variable override is a map with either 1 of 2 keys. +// isFullVariableOverrideDef checks if a mapping with the given keys is a full syntax +// variable override. A full syntax variable override is a map with either 1 of 2 keys. // If it's 2 keys, the keys should be "default" and "type". // If it's 1 key, the key should be one of the following keys: "default", "lookup". -func isFullVariableOverrideDef(v dyn.Value) bool { - mv, ok := v.AsMap() - if !ok { - return false - } - +func isFullVariableOverrideDef(keys []string) bool { // If the map has more than 3 keys, it is not a full variable override. - if mv.Len() > 3 { + if len(keys) > 3 { return false } - for _, keys := range allowedVariableDefinitions { - if len(keys) != mv.Len() { + for _, allowed := range allowedVariableDefinitions { + if len(allowed) != len(keys) { continue } // Check if the keys are the same. match := true - for _, key := range keys { - if _, ok := mv.GetByString(key); !ok { + for _, key := range allowed { + if !slices.Contains(keys, key) { match = false break } @@ -524,75 +347,199 @@ func isFullVariableOverrideDef(v dyn.Value) bool { return false } -// rewriteShorthands performs lightweight rewriting of the configuration -// tree where we allow users to write a shorthand and must rewrite to the full form. -func rewriteShorthands(v dyn.Value) (dyn.Value, error) { - if v.Kind() != dyn.KindMap { - return v, nil +// dealias returns the node an alias refers to, or node itself. +func dealias(node *yaml.Node) *yaml.Node { + for node != nil && node.Kind == yaml.AliasNode { + node = node.Alias } + return node +} - // For each target, rewrite the variables block. - return dyn.Map(v, "targets", dyn.Foreach(func(_ dyn.Path, target dyn.Value) (dyn.Value, error) { - // Confirm it has a variables block. - if target.Get("variables").Kind() == dyn.KindInvalid { - return target, nil +// mappingValue returns the value of key in the YAML mapping node (following aliases). +func mappingValue(node *yaml.Node, key string) *yaml.Node { + node = dealias(node) + if node == nil || node.Kind != yaml.MappingNode { + return nil + } + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + return dealias(node.Content[i+1]) } + } + return nil +} + +// expandMergeKeys returns a copy of the mapping node with "<<" merge keys replaced +// by the pairs they merge in and duplicate keys dropped (the last one wins), so the pairs +// can be rewritten individually. +func expandMergeKeys(node *yaml.Node) *yaml.Node { + var merged, explicit []*yaml.Node + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value != "<<" || node.Content[i].ShortTag() != "!!merge" { + explicit = append(explicit, node.Content[i], node.Content[i+1]) + continue + } + sources := []*yaml.Node{dealias(node.Content[i+1])} + if sources[0].Kind == yaml.SequenceNode { + sources = sources[0].Content + } + for _, src := range sources { + if src = dealias(src); src.Kind == yaml.MappingNode { + merged = append(merged, expandMergeKeys(src).Content...) + } + } + } + // Like the YAML decoder, a later merged map takes precedence over an earlier one and + // keys set explicitly take precedence over all merged maps. + out := *node + out.Content = nil + for _, kvs := range [][]*yaml.Node{merged, explicit} { + for i := 0; i+1 < len(kvs); i += 2 { + setMappingValue(&out, kvs[i], kvs[i+1]) + } + } + return &out +} - // For each variable, normalize its contents if it is a single string. - return dyn.Map(target, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { - switch variable.Kind() { +// setMappingValue replaces the value of key in the mapping node, or appends the pair. +func setMappingValue(node, key, value *yaml.Node) { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key.Value { + node.Content[i+1] = value + return + } + } + node.Content = append(node.Content, key, value) +} + +func mappingKeys(node *yaml.Node) []string { + var keys []string + for i := 0; i+1 < len(node.Content); i += 2 { + keys = append(keys, node.Content[i].Value) + } + return keys +} - case dyn.KindString, dyn.KindBool, dyn.KindFloat, dyn.KindInt: +// mappingNode returns a mapping node with the given keys and values, at the location of at. +func mappingNode(at *yaml.Node, kvs ...any) *yaml.Node { + out := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Line: at.Line, Column: at.Column} + for i := 0; i < len(kvs); i += 2 { + out.Content = append(out.Content, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: kvs[i].(string)}, kvs[i+1].(*yaml.Node)) + } + return out +} + +// effectiveMapping returns the mapping node an alias or mapping node stands for, with merge +// keys expanded and duplicate keys dropped, or nil if it is not a mapping. It never changes +// node: anchors are shared with the rest of the document. +func effectiveMapping(node *yaml.Node) *yaml.Node { + node = dealias(node) + if node == nil || node.Kind != yaml.MappingNode { + return nil + } + return expandMergeKeys(node) +} + +// withMappingValue returns a copy of the mapping node with the value of key replaced. +func withMappingValue(node *yaml.Node, key string, value *yaml.Node) *yaml.Node { + out := *node + out.Content = append([]*yaml.Node(nil), node.Content...) + for i := 0; i+1 < len(out.Content); i += 2 { + if out.Content[i].Value == key { + out.Content[i+1] = value + } + } + return &out +} + +// rewriteShorthands performs lightweight rewriting of the configuration tree where we +// allow users to write a shorthand and must rewrite to the full form: the value of a +// variable in a target. It returns the rewritten tree, sharing what it did not change. +func rewriteShorthands(root *yaml.Node) *yaml.Node { + top := effectiveMapping(root) + if top == nil { + return root + } + targets := effectiveMapping(mappingValue(top, "targets")) + if targets == nil { + return root + } + + // For each target, rewrite the variables block. + changed := false + for i := 1; i < len(targets.Content); i += 2 { + target := effectiveMapping(targets.Content[i]) + if target == nil { + continue + } + variables := effectiveMapping(mappingValue(target, "variables")) + if variables == nil { + continue + } + rewritten := *variables + rewritten.Content = append([]*yaml.Node(nil), variables.Content...) + for j := 0; j+1 < len(rewritten.Content); j += 2 { + name := rewritten.Content[j].Value + variable := dealias(rewritten.Content[j+1]) + switch { + case variable.Kind == yaml.ScalarNode && isShorthandScalar(variable): // Rewrite the variable to a map with a single key called "default". // This conforms to the variable type. Normalization back to the typed // configuration will convert this to a string if necessary. - return dyn.NewValue(map[string]dyn.Value{ - "default": variable, - }, variable.Locations()), nil + rewritten.Content[j+1] = mappingNode(variable, "default", variable) - case dyn.KindMap, dyn.KindSequence: + case variable.Kind == yaml.MappingNode || variable.Kind == yaml.SequenceNode: // If it's a full variable definition, leave it as is. - if isFullVariableOverrideDef(variable) { - return variable, nil + if m := effectiveMapping(variable); m != nil && isFullVariableOverrideDef(mappingKeys(m)) { + continue } // Check if the original definition of variable has a type field. // If it has a type field, it means the shorthand is a value of a complex type. // Type might not be found if the variable overridden in a separate file // and configuration is not merged yet. - typeV, err := dyn.GetByPath(v, p.Append(dyn.Key("type"))) - if err == nil && typeV.MustString() == "complex" { - return dyn.NewValue(map[string]dyn.Value{ - "type": typeV, - "default": variable, - }, variable.Locations()), nil + typeV := mappingValue(mappingValue(mappingValue(top, "variables"), name), "type") + if typeV != nil && typeV.Value == "complex" { + rewritten.Content[j+1] = mappingNode(variable, "type", typeV, "default", variable) + continue } // If it's a shorthand, rewrite it to a full variable definition. - return dyn.NewValue(map[string]dyn.Value{ - "default": variable, - }, variable.Locations()), nil - + rewritten.Content[j+1] = mappingNode(variable, "default", variable) default: - return variable, nil } - })) - })) + } + targets = withMappingValue(targets, targets.Content[i-1].Value, withMappingValue(target, "variables", &rewritten)) + changed = true + } + if !changed { + return root + } + return withMappingValue(top, "targets", targets) +} + +// isShorthandScalar reports whether the scalar is a string, bool, int or float, the kinds a +// variable can be set to directly. A null or a timestamp is not a shorthand. +func isShorthandScalar(n *yaml.Node) bool { + switch n.ShortTag() { + case "!!str", "!!bool", "!!int", "!!float": + return true + default: + return false + } } // validateVariableOverrides checks that all variables specified // in the target override are also defined in the root. -func validateVariableOverrides(root, target dyn.Value) error { - rv, _ := root.Get("variables").AsMap() - tv, _ := target.Get("variables").AsMap() - - // Check that all variables in the target exist in the root. - for _, p := range tv.Pairs() { - if _, ok := rv.GetPair(p.Key); !ok { - return fmt.Errorf("variable %s is not defined but is assigned a value", p.Key.MustString()) +func validateVariableOverrides(root map[string]*variable.Variable, target *Target) error { + if target == nil { + return nil + } + for k := range target.Variables { + if _, ok := root[k]; !ok { + return fmt.Errorf("variable %s is not defined but is assigned a value", k) } } - return nil } @@ -687,18 +634,15 @@ func (r *Root) UpdateSequence(path *structpath.PathNode, sources [][]int) { // GetLocation is [Root.GetLocationOf] for a path string, like main's. func (r Root) GetLocation(path string) diag.Location { - return firstLocation(r.GetLocations(path)) + p, err := structpath.ParsePath(path) + if err != nil { + return diag.Location{} + } + return r.GetLocationOf(p) } // GetLocations is [Root.GetLocationsOf] for a path string, like main's. func (r Root) GetLocations(path string) []diag.Location { - if r.Loading() { - v, err := dyn.Get(r.value, path) - if err != nil { - return nil - } - return v.Locations() - } p, err := structpath.ParsePath(path) if err != nil { return nil @@ -711,9 +655,6 @@ func (r Root) GetLocations(path string) []diag.Location { // Use it to point diagnostics at the most specific place there is; use // [Root.GetLocationsOfExact] to find where a value is written. func (r Root) GetLocationsOf(path *structpath.PathNode) []diag.Location { - if r.Loading() { - return r.GetLocationsOfExact(path) - } return r.locations.Nearest(path) } @@ -725,17 +666,6 @@ func (r Root) GetLocationOf(path *structpath.PathNode) diag.Location { // GetLocationsOfExact returns the locations the configuration value at path is written at, // or none if it was set by a mutator. func (r Root) GetLocationsOfExact(path *structpath.PathNode) []diag.Location { - if r.Loading() { - p, ok := dyn.FromStructPath(path) - if !ok { - return nil - } - v, err := dyn.GetByPath(r.value, p) - if err != nil { - return nil - } - return v.Locations() - } return r.locations.Exact(path) } @@ -897,18 +827,3 @@ func (r *Root) Override(plan *structvar.OverridePlan) error { defer r.store(sv) return sv.Override(plan) } - -// DecodeLoading converts the value at the path of keys in the configuration tree of a -// loading configuration into dst, a pointer to a typed value; dst is left unchanged -// if there is no value there. -func (r *Root) DecodeLoading(dst any, keys ...string) error { - v := r.value - for _, k := range keys { - v = v.Get(k) - } - if !v.IsValid() { - return nil - } - _, _, err := structvar.DecodeDyn(v, dst) - return err -} diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index 4b0a5625fde..6e4bdf3cf37 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -2,11 +2,12 @@ package config import ( "encoding/json" + "maps" "reflect" + "slices" "testing" "github.com/databricks/cli/bundle/config/variable" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -101,9 +102,7 @@ func TestRootMergeTargetOverridesWithMode(t *testing.T) { }, }, } - startLoadingFromTyped(t, root) require.NoError(t, root.MergeTargetOverrides("development")) - finishLoading(t, root) assert.Equal(t, Development, root.Bundle.Mode) } @@ -158,9 +157,7 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) { }, }, } - startLoadingFromTyped(t, root) require.NoError(t, root.MergeTargetOverrides("development")) - finishLoading(t, root) assert.Equal(t, "bar", root.Variables["foo"].Default) assert.Equal(t, "foo var", root.Variables["foo"].Description) @@ -175,103 +172,88 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) { func TestIsFullVariableOverrideDef(t *testing.T) { testCases := []struct { - value dyn.Value + value map[string]any expected bool }{ { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "default": dyn.V("foo"), - "description": dyn.V("foo var"), - }), + value: map[string]any{ + "type": "string", + "default": "foo", + "description": "foo var", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "lookup": dyn.V("foo"), - "description": dyn.V("foo var"), - }), + value: map[string]any{ + "type": "string", + "lookup": "foo", + "description": "foo var", + }, expected: false, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "default": dyn.V("foo"), - }), + value: map[string]any{ + "type": "string", + "default": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "lookup": dyn.V("foo"), - }), + value: map[string]any{ + "type": "string", + "lookup": "foo", + }, expected: false, }, { - value: dyn.V(map[string]dyn.Value{ - "description": dyn.V("string"), - "default": dyn.V("foo"), - }), + value: map[string]any{ + "description": "string", + "default": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "description": dyn.V("string"), - "lookup": dyn.V("foo"), - }), + value: map[string]any{ + "description": "string", + "lookup": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "default": dyn.V("foo"), - }), + value: map[string]any{ + "default": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "lookup": dyn.V("foo"), - }), + value: map[string]any{ + "lookup": "foo", + }, expected: true, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - }), + value: map[string]any{ + "type": "string", + }, expected: false, }, { - value: dyn.V(map[string]dyn.Value{ - "type": dyn.V("string"), - "default": dyn.V("foo"), - "description": dyn.V("foo var"), - "lookup": dyn.V("foo"), - }), + value: map[string]any{ + "type": "string", + "default": "foo", + "description": "foo var", + "lookup": "foo", + }, expected: false, }, } for i, tc := range testCases { - assert.Equal(t, tc.expected, isFullVariableOverrideDef(tc.value), "test case %d", i) + keys := slices.Collect(maps.Keys(tc.value)) + assert.Equal(t, tc.expected, isFullVariableOverrideDef(keys), "test case %d", i) } } -func startLoadingFromTyped(t *testing.T, root *Root) { - require.NoError(t, root.StartLoading()) -} - -func finishLoading(t *testing.T, root *Root) { - diags, err := root.FinishLoading() - require.NoError(t, err) - require.Empty(t, diags) -} - -func loadDyn(t *testing.T, path, content string) *Root { - r, diags := LoadDynFromBytes(path, []byte(content)) - require.NoError(t, diags.Error()) - return r -} - func TestLoadFromBytesNotAMap(t *testing.T) { for _, content := range []string{"hello\n", "- a\n- b\n"} { r, diags := LoadFromBytes("databricks.yml", []byte(content)) @@ -281,15 +263,16 @@ func TestLoadFromBytesNotAMap(t *testing.T) { } func TestMergeComplexVariableSequenceDefaults(t *testing.T) { - a := loadDyn(t, "a.yml", "variables:\n v:\n type: complex\n default: [1, 2]\n") - b := loadDyn(t, "b.yml", "variables:\n v:\n type: complex\n default: [3]\n") + a, diags := LoadFromBytes("a.yml", []byte("variables:\n v:\n type: complex\n default: [1, 2]\n")) + require.NoError(t, diags.Error()) + b, diags := LoadFromBytes("b.yml", []byte("variables:\n v:\n type: complex\n default: [3]\n")) + require.NoError(t, diags.Error()) require.NoError(t, a.Merge(b)) - finishLoading(t, a) assert.Equal(t, []any{1, 2, 3}, a.Variables["v"].Default) } func TestTargetVariableShorthandThroughAliases(t *testing.T) { - r := loadDyn(t, "a.yml", ` + r, diags := LoadFromBytes("a.yml", []byte(` x: &dev dev_value y: &common {b: from_anchor, c: from_anchor} variables: @@ -302,14 +285,19 @@ targets: <<: *common a: *dev c: explicit -`) +`)) + require.NoError(t, diags.Error()) require.NoError(t, r.MergeTargetOverrides("dev")) - finishLoading(t, r) assert.Equal(t, "dev_value", r.Variables["a"].Default) assert.Equal(t, "from_anchor", r.Variables["b"].Default) assert.Equal(t, "explicit", r.Variables["c"].Default) } +func TestNestedYAMLAnchorCycle(t *testing.T) { + _, diags := LoadFromBytes("a.yml", []byte("variables:\n v:\n default: &x {a: [*x]}\n")) + assert.ErrorContains(t, diags.Error(), `cyclic reference to anchor "x"`) +} + func TestSetReferenceAtPointerField(t *testing.T) { r, diags := LoadFromBytes("a.yml", []byte("resources:\n jobs:\n j:\n name: j\n")) require.NoError(t, diags.Error()) @@ -320,11 +308,6 @@ func TestSetReferenceAtPointerField(t *testing.T) { assert.Equal(t, "${var.t}", s) } -func TestNestedYAMLAnchorCycle(t *testing.T) { - _, diags := LoadFromBytes("a.yml", []byte("variables:\n v:\n default: &x {a: [*x]}\n")) - assert.ErrorContains(t, diags.Error(), `cyclic reference to anchor "x"`) -} - func TestRootEmpties(t *testing.T) { r, diags := LoadFromBytes("databricks.yml", []byte(` resources: diff --git a/bundle/config/validate/no_variable_reference_in_resource_key.go b/bundle/config/validate/no_variable_reference_in_resource_key.go index 6d7cb655092..7c6b4f1e8f1 100644 --- a/bundle/config/validate/no_variable_reference_in_resource_key.go +++ b/bundle/config/validate/no_variable_reference_in_resource_key.go @@ -3,10 +3,13 @@ package validate import ( "context" "fmt" + "maps" + "slices" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structvar" ) @@ -25,30 +28,27 @@ func (m *noVariableReferenceInResourceKey) Name() string { func (m *noVariableReferenceInResourceKey) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey(), dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + check := func(prefix *structpath.PathNode, r *config.Resources) { + for _, group := range r.AllResources() { + for _, key := range slices.Sorted(maps.Keys(group.Resources)) { + if !structvar.ContainsVariableReference(key) { + continue + } + p := structpath.NewPath(prefix, group.Description.PluralName, key) + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("resource key %q must not contain variable references", key), + Locations: b.Config.GetLocationsOfExact(p), + Paths: []*structpath.PathNode{p}, + }) + } + } } - for _, pattern := range patterns { - _, err := dyn.MapByPattern( - b.Config.Value(), - pattern, - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - key := p[len(p)-1].Key() - if structvar.ContainsVariableReference(key) { - diags = append(diags, diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("resource key %q must not contain variable references", key), - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), - }) - } - return v, nil - }, - ) - if err != nil { - diags = append(diags, diag.FromErr(err)...) + check(structpath.NewPath(nil, "resources"), &b.Config.Resources) + for _, name := range slices.Sorted(maps.Keys(b.Config.Targets)) { + if t := b.Config.Targets[name]; t != nil && t.Resources != nil { + check(structpath.NewPath(nil, "targets", name, "resources"), t.Resources) } } diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go index 7d2e49569f7..bf51b78fe4a 100644 --- a/bundle/config/validate/unique_resource_keys.go +++ b/bundle/config/validate/unique_resource_keys.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structpath" ) @@ -42,7 +41,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D // Maps of key to the paths and locations the resource / script is defined at. resourceAndScriptMetadata := map[string]*metadata{} - addLocationToMetadata := func(k, prefix string, p dyn.Path, v dyn.Value) { + addLocationToMetadata := func(k string, fullPath *structpath.PathNode) { mv, ok := resourceAndScriptMetadata[k] if !ok { mv = &metadata{ @@ -51,46 +50,22 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D } } - mv.paths = append(mv.paths, dyn.ToStructPath(dyn.NewPath(dyn.Key(prefix)).Append(p...))) - mv.locations = append(mv.locations, v.Locations()...) + mv.paths = append(mv.paths, fullPath) + mv.locations = append(mv.locations, b.Config.GetLocationsOfExact(fullPath)...) resourceAndScriptMetadata[k] = mv } // Gather the paths and locations of all resources - rv := b.Config.Value().Get("resources") - if rv.Kind() != dyn.KindInvalid && rv.Kind() != dyn.KindNil { - _, err := dyn.MapByPattern( - rv, - dyn.NewPattern(dyn.AnyKey(), dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // The key for the resource. Eg: "my_job" for jobs.my_job. - k := p[1].Key() - addLocationToMetadata(k, "resources", p, v) - return v, nil - }, - ) - if err != nil { - return diag.FromErr(err) + for _, group := range b.Config.Resources.AllResources() { + for k := range group.Resources { + addLocationToMetadata(k, structpath.NewPath(nil, "resources", group.Description.PluralName, k)) } } // track locations for all scripts. - sv := b.Config.Value().Get("scripts") - if sv.Kind() != dyn.KindInvalid && sv.Kind() != dyn.KindNil { - _, err := dyn.MapByPattern( - sv, - dyn.NewPattern(dyn.AnyKey()), - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // The key for the script. Eg: "my_script" for scripts.my_script. - k := p[0].Key() - addLocationToMetadata(k, "scripts", p, v) - return v, nil - }, - ) - if err != nil { - return diag.FromErr(err) - } + for k := range b.Config.Scripts { + addLocationToMetadata(k, structpath.NewPath(nil, "scripts", k)) } // If duplicate keys are found, report an error. diff --git a/bundle/configsync/patch_test.go b/bundle/configsync/patch_test.go index 998acc0877f..ef0030f95f8 100644 --- a/bundle/configsync/patch_test.go +++ b/bundle/configsync/patch_test.go @@ -40,7 +40,6 @@ resources: require.NoError(t, err) mutator.DefaultMutators(ctx, b) - mutator.FinishLoading(ctx, b) changes := Changes{ "resources.jobs.test_job": ResourceChanges{ diff --git a/bundle/configsync/resolve_test.go b/bundle/configsync/resolve_test.go index bd458f3347a..9d85fc51119 100644 --- a/bundle/configsync/resolve_test.go +++ b/bundle/configsync/resolve_test.go @@ -30,7 +30,6 @@ func TestResolveSelectors_NoSelectors(t *testing.T) { require.NoError(t, err) mutator.DefaultMutators(ctx, b) - mutator.FinishLoading(ctx, b) result, err := resolveSelectors("resources.jobs.test_job.name", b, OperationReplace) require.NoError(t, err) @@ -56,7 +55,6 @@ func TestResolveSelectors_NumericIndices(t *testing.T) { require.NoError(t, err) mutator.DefaultMutators(ctx, b) - mutator.FinishLoading(ctx, b) result, err := resolveSelectors("resources.jobs.test_job.tasks[0].task_key", b, OperationReplace) require.NoError(t, err) @@ -90,7 +88,6 @@ func TestResolveSelectors_KeyValueSelector(t *testing.T) { require.NoError(t, err) mutator.DefaultMutators(ctx, b) - mutator.FinishLoading(ctx, b) result, err := resolveSelectors("resources.jobs.test_job.tasks[task_key='main'].notebook_task.notebook_path", b, OperationReplace) require.NoError(t, err) @@ -121,7 +118,6 @@ func TestResolveSelectors_SelectorNotFound(t *testing.T) { require.NoError(t, err) mutator.DefaultMutators(ctx, b) - mutator.FinishLoading(ctx, b) _, err = resolveSelectors("resources.jobs.test_job.tasks[task_key='nonexistent'].notebook_task.notebook_path", b, OperationReplace) require.Error(t, err) @@ -145,7 +141,6 @@ func TestResolveSelectors_SelectorOnNonArray(t *testing.T) { require.NoError(t, err) mutator.DefaultMutators(ctx, b) - mutator.FinishLoading(ctx, b) _, err = resolveSelectors("resources.jobs.test_job[task_key='main'].name", b, OperationReplace) require.Error(t, err) @@ -177,7 +172,6 @@ func TestResolveSelectors_NestedSelectors(t *testing.T) { require.NoError(t, err) mutator.DefaultMutators(ctx, b) - mutator.FinishLoading(ctx, b) result, err := resolveSelectors("resources.jobs.test_job.tasks[task_key='main'].libraries[0].pypi.package", b, OperationReplace) require.NoError(t, err) @@ -204,7 +198,6 @@ func TestResolveSelectors_WildcardNotSupported(t *testing.T) { require.NoError(t, err) mutator.DefaultMutators(ctx, b) - mutator.FinishLoading(ctx, b) _, err = resolveSelectors("resources.jobs.test_job.tasks.*.task_key", b, OperationReplace) require.Error(t, err) diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 2a1412c285b..5d5c3c46471 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -122,11 +122,10 @@ func LoadPreResolvedConfig(ctx context.Context, b *bundle.Bundle) structvar.View } mutator.DefaultMutators(ctx, fresh) if target := b.Config.Bundle.Target; target != "" { - if fresh.Config.HasTarget(target) { + if _, ok := fresh.Config.Targets[target]; ok { bundle.ApplyContext(ctx, fresh, mutator.SelectTarget(target)) } } - mutator.FinishLoading(ctx, fresh) // Keyed sequences merge in the initialize phase, which this reload skips. Without // them the sequences here stay in file order while the change paths address the diff --git a/bundle/internal/bundletest/component_benchmark_test.go b/bundle/internal/bundletest/component_benchmark_test.go index b7b33956540..4ec348c0933 100644 --- a/bundle/internal/bundletest/component_benchmark_test.go +++ b/bundle/internal/bundletest/component_benchmark_test.go @@ -75,8 +75,8 @@ func BenchmarkComponentExpandGlobReferences(b *testing.B) { benchmarkMutator(b, libraries.ExpandGlobReferences()) } -// componentLoadInput returns a configuration file with 1000 jobs (JSON is valid YAML). -func componentLoadInput(b *testing.B) []byte { +// BenchmarkComponentLoad loads a configuration file with 1000 jobs (JSON is valid YAML). +func BenchmarkComponentLoad(b *testing.B) { var job map[string]any require.NoError(b, json.Unmarshal([]byte(jobExample), &job)) jobs := map[string]any{} @@ -89,37 +89,9 @@ func componentLoadInput(b *testing.B) []byte { }) require.NoError(b, err) require.True(b, strings.HasPrefix(string(raw), "{")) - return raw -} -// BenchmarkComponentLoad loads a configuration file with 1000 jobs into the typed -// configuration: [BenchmarkComponentLoadDyn] plus [BenchmarkComponentFinishLoading]. -func BenchmarkComponentLoad(b *testing.B) { - raw := componentLoadInput(b) for b.Loop() { _, diags := config.LoadFromBytes("databricks.yml", raw) require.False(b, diags.HasError(), "%v", diags) } } - -// BenchmarkComponentLoadDyn loads a configuration file with 1000 jobs into a loading -// configuration (configuration tree), like the load phase does for every file. -func BenchmarkComponentLoadDyn(b *testing.B) { - raw := componentLoadInput(b) - for b.Loop() { - _, diags := config.LoadDynFromBytes("databricks.yml", raw) - require.False(b, diags.HasError(), "%v", diags) - } -} - -// BenchmarkComponentFinishLoading converts a loading configuration with 1000 jobs into -// the typed configuration, which the load phase does once. -func BenchmarkComponentFinishLoading(b *testing.B) { - r, diags := config.LoadDynFromBytes("databricks.yml", componentLoadInput(b)) - require.False(b, diags.HasError(), "%v", diags) - for b.Loop() { - c := *r - _, err := c.FinishLoading() - require.NoError(b, err) - } -} diff --git a/bundle/internal/bundletest/loading.go b/bundle/internal/bundletest/loading.go deleted file mode 100644 index b2bb93b890d..00000000000 --- a/bundle/internal/bundletest/loading.go +++ /dev/null @@ -1,20 +0,0 @@ -package bundletest - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" - "github.com/stretchr/testify/require" -) - -// ApplyLoading applies mutators of the load phase, which work on the configuration -// tree, to b (whose configuration may be typed) and converts the result into the -// typed configuration, like the load phase does. -func ApplyLoading(t testing.TB, b *bundle.Bundle, mutators ...bundle.Mutator) diag.Diagnostics { - require.NoError(t, b.Config.StartLoading()) - diags := bundle.ApplySeq(t.Context(), b, mutators...) - finishDiags, err := b.Config.FinishLoading() - require.NoError(t, err) - return diags.Extend(finishDiags) -} diff --git a/bundle/phases/load.go b/bundle/phases/load.go index b0c158d74a5..6fa1106b3bf 100644 --- a/bundle/phases/load.go +++ b/bundle/phases/load.go @@ -11,22 +11,15 @@ import ( // The load phase loads configuration from disk and performs // lightweight preprocessing (anything that can be done without network I/O). -// -// The configuration is loaded, merged with its includes and its target as a -// configuration tree, then converted once into the typed configuration -// (see [mutator.FinishLoading]); all later phases use the typed configuration. func Load(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: load") mutator.DefaultMutators(ctx, b) - mutator.FinishLoading(ctx, b) } func LoadDefaultTarget(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: load") - defer mutator.FinishLoading(ctx, b) - mutator.DefaultMutators(ctx, b) if logdiag.HasError(ctx) { return @@ -38,8 +31,6 @@ func LoadDefaultTarget(ctx context.Context, b *bundle.Bundle) { func LoadNamedTarget(ctx context.Context, b *bundle.Bundle, target string) { log.Info(ctx, "Phase: load") - defer mutator.FinishLoading(ctx, b) - mutator.DefaultMutators(ctx, b) if logdiag.HasError(ctx) { return diff --git a/bundle/scripts/scripts.go b/bundle/scripts/scripts.go index 85fe77a7184..cca108ec514 100644 --- a/bundle/scripts/scripts.go +++ b/bundle/scripts/scripts.go @@ -97,13 +97,6 @@ func executeHook(ctx context.Context, executor *exec.Executor, command config.Co } func getCommand(b *bundle.Bundle, hook config.ScriptHook) config.Command { - // The preinit script runs while the configuration is loading. - if b.Config.Loading() { - var command config.Command - _ = b.Config.DecodeLoading(&command, "experimental", "scripts", string(hook)) - return command - } - if b.Config.Experimental == nil || b.Config.Experimental.Scripts == nil { return "" } diff --git a/bundle/tests/suggest_target_test.go b/bundle/tests/suggest_target_test.go index 18f7a5ae9dc..ca678a61c0a 100644 --- a/bundle/tests/suggest_target_test.go +++ b/bundle/tests/suggest_target_test.go @@ -3,15 +3,16 @@ package config_tests import ( "testing" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/internal/bundletest" "github.com/stretchr/testify/require" ) func TestSuggestTargetIfWrongPassed(t *testing.T) { b := load(t, "target_overrides/workspace") - diags := bundletest.ApplyLoading(t, b, mutator.SelectTarget("incorrect")) + ctx := t.Context() + diags := bundle.Apply(ctx, b, mutator.SelectTarget("incorrect")) err := diags.Error() require.Error(t, err) require.Contains(t, err.Error(), "Available targets:") diff --git a/libs/dyn/convert/end_to_end_test.go b/libs/dyn/convert/end_to_end_test.go deleted file mode 100644 index e3ea273cb04..00000000000 --- a/libs/dyn/convert/end_to_end_test.go +++ /dev/null @@ -1,171 +0,0 @@ -package convert - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func assertFromTypedToTypedEqual[T any](t *testing.T, src T) { - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - - var dst T - err = ToTyped(&dst, nv) - require.NoError(t, err) - assert.Equal(t, src, dst) -} - -func TestAdditional(t *testing.T) { - type StructType struct { - Str string `json:"str"` - } - - type Tmp struct { - MapToPointer map[string]*string `json:"map_to_pointer"` - SliceOfPointer []*string `json:"slice_of_pointer"` - NestedStruct StructType `json:"nested_struct"` - } - - t.Run("nil", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{}) - }) - - t.Run("empty map", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{ - MapToPointer: map[string]*string{}, - }) - }) - - t.Run("map with empty string value", func(t *testing.T) { - s := "" - assertFromTypedToTypedEqual(t, Tmp{ - MapToPointer: map[string]*string{ - "key": &s, - }, - }) - }) - - t.Run("map with nil value", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{ - MapToPointer: map[string]*string{ - "key": nil, - }, - }) - }) - - t.Run("empty slice", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{ - SliceOfPointer: []*string{}, - }) - }) - - t.Run("slice with nil value", func(t *testing.T) { - assertFromTypedToTypedEqual(t, Tmp{ - SliceOfPointer: []*string{nil}, - }) - }) - - t.Run("pointer to a empty string", func(t *testing.T) { - s := "" - assertFromTypedToTypedEqual(t, &s) - }) - - t.Run("nil pointer", func(t *testing.T) { - var s *string - assertFromTypedToTypedEqual(t, s) - }) - - t.Run("pointer to struct with scalar values", func(t *testing.T) { - s := "" - type foo struct { - A string `json:"a"` - B int `json:"b"` - C bool `json:"c"` - D *string `json:"d"` - } - assertFromTypedToTypedEqual(t, &foo{ - A: "a", - B: 1, - C: true, - D: &s, - }) - assertFromTypedToTypedEqual(t, &foo{ - A: "", - B: 0, - C: false, - D: nil, - }) - }) - - t.Run("map with scalar values", func(t *testing.T) { - assertFromTypedToTypedEqual(t, map[string]string{ - "a": "a", - "b": "b", - "c": "", - }) - assertFromTypedToTypedEqual(t, map[string]int{ - "a": 1, - "b": 0, - "c": 2, - }) - }) -} - -func TestEndToEndForceSendFields(t *testing.T) { - type Inner struct { - InnerField string `json:"inner_field"` - ForceSendFields []string `json:"-"` - } - type Outer struct { - OuterField string `json:"outer_field"` - Inner - } - - // Test with zero value in embedded struct - src := Outer{ - OuterField: "outer_value", - Inner: Inner{ - InnerField: "", // Zero value - ForceSendFields: []string{"InnerField"}, // Should be preserved - }, - } - - assertFromTypedToTypedEqual(t, src) -} - -func TestEndToEndPointerForceSendFields(t *testing.T) { - type NewCluster struct { - NumWorkers int `json:"num_workers"` - SparkVersion string `json:"spark_version"` - ForceSendFields []string `json:"-"` - } - type JobCluster struct { - JobClusterKey string `json:"job_cluster_key"` - NewCluster *NewCluster `json:"new_cluster"` - } - type JobSettings struct { - JobClusters []JobCluster `json:"job_clusters"` - Name string `json:"name"` - ForceSendFields []string `json:"-"` - } - - // Test with zero value in pointer embedded struct (like acceptance test) - src := JobSettings{ - Name: "test-job", - JobClusters: []JobCluster{ - { - JobClusterKey: "key", - NewCluster: &NewCluster{ - NumWorkers: 0, // Zero value - SparkVersion: "13.3.x-scala2.12", - ForceSendFields: []string{"NumWorkers"}, // Should be preserved - }, - }, - }, - } - - assertFromTypedToTypedEqual(t, src) -} diff --git a/libs/dyn/convert/error.go b/libs/dyn/convert/error.go deleted file mode 100644 index d3770d82d74..00000000000 --- a/libs/dyn/convert/error.go +++ /dev/null @@ -1,16 +0,0 @@ -package convert - -import ( - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -type TypeError struct { - value dyn.Value - msg string -} - -func (e TypeError) Error() string { - return fmt.Sprintf("%s: %s", e.value.Location(), e.msg) -} diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go deleted file mode 100644 index 66451124293..00000000000 --- a/libs/dyn/convert/from_typed.go +++ /dev/null @@ -1,362 +0,0 @@ -package convert - -import ( - "fmt" - "reflect" - "slices" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" -) - -type fromTypedOptions int - -const ( - // If this flag is set, zero values in the typed representation are resolved to - // the equivalent zero value in the dynamic representation. - // If it is not set, zero values resolve to [dyn.NilValue]. - // - // This flag exists to reconcile default values in Go being zero values with values - // being intentionally set to their zero value. We capture zero values in the dynamic - // configuration if they are 1) behind a pointer, 2) a map value, 3) a slice element, - // in the typed configuration. - includeZeroValues fromTypedOptions = 1 << iota -) - -// FromTyped converts changes made in the typed structure w.r.t. the configuration value -// back to the configuration value, retaining existing location information where possible. -// -// It uses the reference value both for location information and to determine if the typed -// value was changed or not. For example, if a struct-by-value field is nil in the reference -// it will be zero-valued in the typed configuration. If it remains zero-valued, this -// function will still emit a nil value in the dynamic representation. -func FromTyped(src any, ref dyn.Value) (dyn.Value, error) { - return fromTyped(src, ref) -} - -// Private implementation of FromTyped that allows for additional options not exposed -// in the public API. -func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - srcv := reflect.ValueOf(src) - - // Dereference pointer if necessary - for srcv.Kind() == reflect.Pointer { - if srcv.IsNil() { - return dyn.NilValue.WithLocations(ref.Locations()), nil - } - srcv = srcv.Elem() - - // If a pointer to a type points to a zero value, we should include - // that zero value in the dynamic representation. - // This is because by default a pointer is nil in Go, and it not being nil - // indicates its value was intentionally set to zero. - if !slices.Contains(options, includeZeroValues) { - options = append(options, includeZeroValues) - } - } - - var v dyn.Value - var err error - switch srcv.Kind() { - case reflect.Struct: - // Handle SDK native types using JSON marshaling. - if slices.Contains(sdkNativeTypes, srcv.Type()) { - v, err = fromTypedSDKNative(srcv, ref, options...) - } else { - v, err = fromTypedStruct(srcv, ref, options...) - } - case reflect.Map: - v, err = fromTypedMap(srcv, ref) - case reflect.Slice: - v, err = fromTypedSlice(srcv, ref) - case reflect.String: - v, err = fromTypedString(srcv, ref, options...) - case reflect.Bool: - v, err = fromTypedBool(srcv, ref, options...) - case reflect.Int, reflect.Int32, reflect.Int64: - v, err = fromTypedInt(srcv, ref, options...) - case reflect.Float32, reflect.Float64: - v, err = fromTypedFloat(srcv, ref, options...) - case reflect.Invalid: - // If the value is untyped and not set (e.g. any type with nil value), we return nil. - v, err = dyn.NilValue, nil - default: - return dyn.InvalidValue, fmt.Errorf("unsupported type: %s", srcv.Kind()) - } - - // Ensure the location metadata is retained. - if err != nil { - return dyn.InvalidValue, err - } - return v.WithLocations(ref.Locations()), err -} - -func fromTypedStruct(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - // Check that the reference value is compatible or nil. - switch ref.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - case dyn.KindMap, dyn.KindNil: - default: - return dyn.InvalidValue, fmt.Errorf("cannot convert struct field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) - } - - refm, _ := ref.AsMap() - out := dyn.NewMapping() - info := getStructInfo(src.Type()) - - for _, fieldval := range info.FieldValues(src) { - k := fieldval.Key - v := fieldval.Value - isForced := fieldval.IsForced - pair, ok := refm.GetPairByString(k) - refloc := pair.Key.Locations() - refv := pair.Value - - // Use nil reference if there is no reference for this key - if !ok { - refloc = nil - refv = dyn.NilValue - } - - var options []fromTypedOptions - if v.Kind() == reflect.Interface { - options = append(options, includeZeroValues) - } - - // Convert the field taking into account the reference value (may be equal to config.NilValue). - nv, err := fromTyped(v.Interface(), refv, options...) - if err != nil { - return dyn.InvalidValue, err - } - - // Either if the key was set in the reference, the field is not zero-valued, OR it's forced - if ok || nv.Kind() != dyn.KindNil || isForced { - // If v isZero, it could be because it's a variable reference; so we check that nv is zero as well - // BUT: always include if it's forced - if v.Kind() != reflect.Struct && v.IsZero() && nv.IsZero() && !info.ForceEmpty[k] && !isForced { - continue - } - - // If the field is forced but nv is nil, convert it to the appropriate zero value - if isForced && nv.Kind() == dyn.KindNil { - // Convert the zero value using proper recursive conversion instead of dyn.V() directly - // This prevents "not handled" panics for complex types like structs, slices, maps, etc. - // Use refv to preserve location information from the original reference - var err error - nv, err = fromTyped(v.Interface(), refv, includeZeroValues) - if err != nil { - return dyn.InvalidValue, err - } - } - - out.SetLoc(k, refloc, nv) - } - } - - // Return the new mapping if: - // 1. The mapping has entries (i.e. the struct was not empty). - // 2. The reference is a map (i.e. the struct was and still is empty). - // 3. The "includeZeroValues" option is set (i.e. the struct is a non-nil pointer). - if out.Len() > 0 || ref.Kind() == dyn.KindMap || slices.Contains(options, includeZeroValues) { - return dyn.V(out), nil - } - - // Otherwise, return nil. - return dyn.NilValue, nil -} - -func fromTypedMap(src reflect.Value, ref dyn.Value) (dyn.Value, error) { - // Check that the reference value is compatible or nil. - switch ref.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - case dyn.KindMap, dyn.KindNil: - default: - return dyn.InvalidValue, fmt.Errorf("cannot convert map field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) - } - - // Return nil if the map is nil. - if src.IsNil() { - return dyn.NilValue, nil - } - - refm, _ := ref.AsMap() - out := dyn.NewMapping() - iter := src.MapRange() - for iter.Next() { - k := iter.Key().String() - v := iter.Value() - pair, ok := refm.GetPairByString(k) - refloc := pair.Key.Locations() - refv := pair.Value - - // Use nil reference if there is no reference for this key - if !ok { - refloc = nil - refv = dyn.NilValue - } - - // Convert entry taking into account the reference value (may be equal to dyn.NilValue). - nv, err := fromTyped(v.Interface(), refv, includeZeroValues) - if err != nil { - return dyn.InvalidValue, err - } - - // Every entry is represented, even if it is a nil. - // Otherwise, a map with zero-valued structs would yield a nil as well. - out.SetLoc(k, refloc, nv) - } - - return dyn.V(out), nil -} - -func fromTypedSlice(src reflect.Value, ref dyn.Value) (dyn.Value, error) { - // Check that the reference value is compatible or nil. - switch ref.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - case dyn.KindSequence, dyn.KindNil: - default: - return dyn.InvalidValue, fmt.Errorf("cannot convert slice field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) - } - - // Return nil if the slice is nil. - if src.IsNil() { - return dyn.NilValue, nil - } - - out := make([]dyn.Value, src.Len()) - for i := range src.Len() { - v := src.Index(i) - refv := ref.Index(i) - - // Use nil reference if there is no reference for this index. - if refv.Kind() == dyn.KindInvalid { - refv = dyn.NilValue - } - - // Convert entry taking into account the reference value (may be equal to dyn.NilValue). - nv, err := fromTyped(v.Interface(), refv, includeZeroValues) - if err != nil { - return dyn.InvalidValue, err - } - - out[i] = nv - } - - return dyn.V(out), nil -} - -func fromTypedString(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - switch ref.Kind() { - case dyn.KindString: - value := src.String() - if value == ref.MustString() { - return ref, nil - } - - return dyn.V(value), nil - case dyn.KindNil: - // This field is not set in the reference. We set it to nil if it's zero - // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - return dyn.V(src.String()), nil - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("cannot convert string field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) -} - -func fromTypedBool(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - switch ref.Kind() { - case dyn.KindBool: - value := src.Bool() - if value == ref.MustBool() { - return ref, nil - } - return dyn.V(value), nil - case dyn.KindNil: - // This field is not set in the reference. We set it to nil if it's zero - // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - return dyn.V(src.Bool()), nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("cannot convert bool field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) -} - -func fromTypedInt(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - switch ref.Kind() { - case dyn.KindInt: - value := src.Int() - if value == ref.MustInt() { - return ref, nil - } - return dyn.V(value), nil - case dyn.KindNil: - // This field is not set in the reference. We set it to nil if it's zero - // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - return dyn.V(src.Int()), nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("cannot convert int field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) -} - -func fromTypedFloat(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - switch ref.Kind() { - case dyn.KindFloat: - value := src.Float() - if value == ref.MustFloat() { - return ref, nil - } - return dyn.V(value), nil - case dyn.KindNil: - // This field is not set in the reference. We set it to nil if it's zero - // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - return dyn.V(src.Float()), nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("cannot convert float field to dynamic type %#v: src=%#v ref=%#v", ref.Kind().String(), src, ref.AsAny()) -} diff --git a/libs/dyn/convert/from_typed_test.go b/libs/dyn/convert/from_typed_test.go deleted file mode 100644 index 3420c46cbd7..00000000000 --- a/libs/dyn/convert/from_typed_test.go +++ /dev/null @@ -1,927 +0,0 @@ -package convert - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFromTypedStructZeroFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{} - - // For an empty struct with a nil reference we expect a nil. - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - - // For an empty struct with a non-nil reference we expect an empty map. - nv, err = FromTyped(src, dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{}), nv) -} - -func TestFromTypedStructPointerZeroFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - var src *Tmp - var nv dyn.Value - var err error - - // For a nil pointer with a nil reference we expect a nil. - src = nil - nv, err = FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - - // For a nil pointer with a non-nil reference we expect a nil. - src = nil - nv, err = FromTyped(src, dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - - // For an initialized pointer with a nil reference we expect an empty map. - src = &Tmp{} - nv, err = FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{}), nv) - - // For an initialized pointer with a non-nil reference we expect an empty map. - src = &Tmp{} - nv, err = FromTyped(src, dyn.V(map[string]dyn.Value{})) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{}), nv) -} - -func TestFromTypedStructNilFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - // For a zero value struct with a reference containing nil fields we expect the nils to be retained. - src := Tmp{} - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NilValue, - "bar": dyn.NilValue, - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.NilValue, - "bar": dyn.NilValue, - }), nv) -} - -func TestFromTypedStructSetFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{ - Foo: "foo", - Bar: "bar", - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V("bar"), - }), nv) -} - -func TestFromTypedStructSetFieldsIntoExistingFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{ - Foo: "foo", - Bar: "bar", - } - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo_old"), - "bar": dyn.V("bar_old"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V("bar"), - }), nv) -} - -func TestFromTypedStructZeroFieldsIntoExistingFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - BazBugz string `json:"baz_bugs,omitempty"` - ForceSendFields []string `json:"-"` - } - - src := Tmp{ForceSendFields: []string{"BazBugz"}} - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo_old"), - "bar": dyn.V("bar_old"), - "baz_bugs": dyn.V("baz_old"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(""), - "baz_bugs": dyn.V(""), - }), nv) -} - -func TestFromTypedStructZeroFieldsIntoExistingFieldsFloat64(t *testing.T) { - type Tmp struct { - Foo float64 `json:"foo"` - Bar float64 `json:"bar,omitempty"` - BazBugz float64 `json:"baz_bugs,omitempty"` - ForceSendFields []string `json:"-"` - } - - src := Tmp{ForceSendFields: []string{"BazBugz"}} - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(1.2), - "bar": dyn.V(1.2), - "baz_bugs": dyn.V(1.2), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(0.0), - "baz_bugs": dyn.V(0.0), - }), nv) -} - -func TestFromTypedStructSetFieldsRetainLocation(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{ - Foo: "bar", - Bar: "qux", - } - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", []diag.Location{{File: "foo"}}), - "bar": dyn.NewValue("baz", []diag.Location{{File: "bar"}}), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - - // Assert foo and bar have retained their location. - assert.Equal(t, dyn.NewValue("bar", []diag.Location{{File: "foo"}}), nv.Get("foo")) - assert.Equal(t, dyn.NewValue("qux", []diag.Location{{File: "bar"}}), nv.Get("bar")) -} - -func TestFromTypedStringMapWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := map[string]string{ - "foo": "", - "bar": "fuzz", - } - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(""), - "bar": dyn.V("fuzz"), - }), nv) -} - -func TestFromTypedStringSliceWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := []string{"a", "", "c"} - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V("a"), dyn.V(""), dyn.V("c"), - }), nv) -} - -func TestFromTypedStringStructWithZeroValue(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - ref := dyn.NilValue - src := Tmp{ - Foo: "foo", - Bar: "", - } - - // Note, the zero value is not included in the output. - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - }), nv) -} - -func TestFromTypedBoolMapWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := map[string]bool{ - "foo": false, - "bar": true, - } - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(false), - "bar": dyn.V(true), - }), nv) -} - -func TestFromTypedBoolSliceWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := []bool{true, false, true} - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V(true), dyn.V(false), dyn.V(true), - }), nv) -} - -func TestFromTypedBoolStructWithZeroValue(t *testing.T) { - type Tmp struct { - Foo bool `json:"foo"` - Bar bool `json:"bar"` - } - - ref := dyn.NilValue - src := Tmp{ - Foo: true, - Bar: false, - } - - // Note, the zero value is not included in the output. - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(true), - }), nv) -} - -func TestFromTypedIntMapWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := map[string]int{ - "foo": 0, - "bar": 1, - } - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(int64(0)), - "bar": dyn.V(int64(1)), - }), nv) -} - -func TestFromTypedIntSliceWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := []int{1, 0, 2} - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V(int64(1)), dyn.V(int64(0)), dyn.V(int64(2)), - }), nv) -} - -func TestFromTypedIntStructWithZeroValue(t *testing.T) { - type Tmp struct { - Foo int `json:"foo"` - Bar int `json:"bar"` - } - - ref := dyn.NilValue - src := Tmp{ - Foo: 1, - Bar: 0, - } - - // Note, the zero value is not included in the output. - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(int64(1)), - }), nv) -} - -func TestFromTypedFloatMapWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := map[string]float64{ - "foo": 0.0, - "bar": 1.0, - } - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(0.0), - "bar": dyn.V(1.0), - }), nv) -} - -func TestFromTypedFloatSliceWithZeroValue(t *testing.T) { - ref := dyn.NilValue - src := []float64{1.0, 0.0, 2.0} - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V(1.0), dyn.V(0.0), dyn.V(2.0), - }), nv) -} - -func TestFromTypedFloatStructWithZeroValue(t *testing.T) { - type Tmp struct { - Foo float64 `json:"foo"` - Bar float64 `json:"bar"` - } - - ref := dyn.NilValue - src := Tmp{ - Foo: 1.0, - Bar: 0.0, - } - - // Note, the zero value is not included in the output. - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(1.0), - }), nv) -} - -func TestFromTypedMapNil(t *testing.T) { - var src map[string]string = nil - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedMapEmpty(t *testing.T) { - src := map[string]string{} - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{}), nv) -} - -func TestFromTypedMapNonEmpty(t *testing.T) { - src := map[string]string{ - "foo": "foo", - "bar": "bar", - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V("bar"), - }), nv) -} - -func TestFromTypedMapNonEmptyRetainLocation(t *testing.T) { - src := map[string]string{ - "foo": "bar", - "bar": "qux", - } - - ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", []diag.Location{{File: "foo"}}), - "bar": dyn.NewValue("baz", []diag.Location{{File: "bar"}}), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - - // Assert foo and bar have retained their locations. - assert.Equal(t, dyn.NewValue("bar", []diag.Location{{File: "foo"}}), nv.Get("foo")) - assert.Equal(t, dyn.NewValue("qux", []diag.Location{{File: "bar"}}), nv.Get("bar")) -} - -func TestFromTypedMapFieldWithZeroValue(t *testing.T) { - src := map[string]string{ - "foo": "", - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V(""), - }), nv) -} - -func TestFromTypedSliceNil(t *testing.T) { - var src []string = nil - - ref := dyn.V([]dyn.Value{ - dyn.V("bar"), - dyn.V("baz"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedSliceEmpty(t *testing.T) { - //nolint:gocritic - src := []string{} - - ref := dyn.V([]dyn.Value{ - dyn.V("bar"), - dyn.V("baz"), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{}), nv) -} - -func TestFromTypedSliceNonEmpty(t *testing.T) { - src := []string{ - "foo", - "bar", - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }), nv) -} - -func TestFromTypedSliceNonEmptyRetainLocation(t *testing.T) { - src := []string{ - "foo", - "bar", - } - - ref := dyn.V([]dyn.Value{ - dyn.NewValue("foo", []diag.Location{{File: "foo"}}), - dyn.NewValue("bar", []diag.Location{{File: "bar"}}), - }) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - - // Assert foo and bar have retained their locations. - assert.Equal(t, dyn.NewValue("foo", []diag.Location{{File: "foo"}}), nv.Index(0)) - assert.Equal(t, dyn.NewValue("bar", []diag.Location{{File: "bar"}}), nv.Index(1)) -} - -func TestFromTypedStringEmpty(t *testing.T) { - var src string - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedStringEmptyOverwrite(t *testing.T) { - var src string - ref := dyn.V("old") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(""), nv) -} - -func TestFromTypedStringNonEmpty(t *testing.T) { - src := "new" - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("new"), nv) -} - -func TestFromTypedStringNonEmptyOverwrite(t *testing.T) { - src := "new" - ref := dyn.V("old") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("new"), nv) -} - -func TestFromTypedStringRetainsLocations(t *testing.T) { - ref := dyn.NewValue("foo", []diag.Location{{File: "foo"}}) - - // case: value has not been changed - src := "foo" - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue("foo", []diag.Location{{File: "foo"}}), nv) - - // case: value has been changed - src = "bar" - nv, err = FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue("bar", []diag.Location{{File: "foo"}}), nv) -} - -func TestFromTypedStringTypeError(t *testing.T) { - src := "foo" - ref := dyn.V(1234) - _, err := FromTyped(src, ref) - require.Error(t, err) -} - -func TestFromTypedBoolEmpty(t *testing.T) { - var src bool - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedBoolEmptyOverwrite(t *testing.T) { - var src bool - ref := dyn.V(true) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(false), nv) -} - -func TestFromTypedBoolNonEmpty(t *testing.T) { - src := true - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(true), nv) -} - -func TestFromTypedBoolNonEmptyOverwrite(t *testing.T) { - src := true - ref := dyn.V(false) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(true), nv) -} - -func TestFromTypedBoolRetainsLocations(t *testing.T) { - ref := dyn.NewValue(true, []diag.Location{{File: "foo"}}) - - // case: value has not been changed - src := true - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(true, []diag.Location{{File: "foo"}}), nv) - - // case: value has been changed - src = false - nv, err = FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(false, []diag.Location{{File: "foo"}}), nv) -} - -func TestFromTypedBoolVariableReference(t *testing.T) { - src := true - ref := dyn.V("${var.foo}") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("${var.foo}"), nv) -} - -func TestFromTypedBoolTypeError(t *testing.T) { - src := true - ref := dyn.V("string") - _, err := FromTyped(src, ref) - require.Error(t, err) -} - -func TestFromTypedIntEmpty(t *testing.T) { - var src int - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedIntEmptyOverwrite(t *testing.T) { - var src int - ref := dyn.V(1234) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(int64(0)), nv) -} - -func TestFromTypedIntNonEmpty(t *testing.T) { - src := 1234 - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(int64(1234)), nv) -} - -func TestFromTypedIntNonEmptyOverwrite(t *testing.T) { - src := 1234 - ref := dyn.V(1233) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(int64(1234)), nv) -} - -func TestFromTypedIntRetainsLocations(t *testing.T) { - ref := dyn.NewValue(1234, []diag.Location{{File: "foo"}}) - - // case: value has not been changed - src := 1234 - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1234, []diag.Location{{File: "foo"}}), nv) - - // case: value has been changed - src = 1235 - nv, err = FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(int64(1235), []diag.Location{{File: "foo"}}), nv) -} - -func TestFromTypedIntVariableReference(t *testing.T) { - src := 1234 - ref := dyn.V("${var.foo}") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("${var.foo}"), nv) -} - -func TestFromTypedIntTypeError(t *testing.T) { - src := 1234 - ref := dyn.V("string") - _, err := FromTyped(src, ref) - require.Error(t, err) -} - -func TestFromTypedFloatEmpty(t *testing.T) { - var src float64 - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedFloatEmptyOverwrite(t *testing.T) { - var src float64 - ref := dyn.V(1.23) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(0.0), nv) -} - -func TestFromTypedFloatNonEmpty(t *testing.T) { - src := 1.23 - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(1.23), nv) -} - -func TestFromTypedFloatNonEmptyOverwrite(t *testing.T) { - src := 1.23 - ref := dyn.V(1.24) - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V(1.23), nv) -} - -func TestFromTypedFloatRetainsLocations(t *testing.T) { - var src float64 - ref := dyn.NewValue(1.23, []diag.Location{{File: "foo"}}) - - // case: value has not been changed - src = 1.23 - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1.23, []diag.Location{{File: "foo"}}), nv) - - // case: value has been changed - src = 1.24 - nv, err = FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1.24, []diag.Location{{File: "foo"}}), nv) -} - -func TestFromTypedFloatVariableReference(t *testing.T) { - src := 1.23 - ref := dyn.V("${var.foo}") - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("${var.foo}"), nv) -} - -func TestFromTypedFloatTypeError(t *testing.T) { - src := 1.23 - ref := dyn.V("string") - _, err := FromTyped(src, ref) - require.Error(t, err) -} - -func TestFromTypedAny(t *testing.T) { - type Tmp struct { - Foo any `json:"foo"` - Bar any `json:"bar"` - Foz any `json:"foz"` - Baz any `json:"baz"` - } - - src := Tmp{ - Foo: "foo", - Bar: false, - Foz: 0, - Baz: map[string]any{ - "foo": "foo", - "bar": 1234, - "qux": 0, - "nil": nil, - }, - } - - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V(false), - "foz": dyn.V(int64(0)), - "baz": dyn.V(map[string]dyn.Value{ - "foo": dyn.V("foo"), - "bar": dyn.V(int64(1234)), - "qux": dyn.V(int64(0)), - "nil": dyn.V(nil), - }), - }), nv) -} - -func TestFromTypedAnyNil(t *testing.T) { - var src any = nil - ref := dyn.NilValue - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestFromTypedNilPointerRetainsLocations(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - var src *Tmp - ref := dyn.NewValue(nil, []diag.Location{{File: "foobar"}}) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, []diag.Location{{File: "foobar"}}), nv) -} - -func TestFromTypedNilMapRetainsLocation(t *testing.T) { - var src map[string]string - ref := dyn.NewValue(nil, []diag.Location{{File: "foobar"}}) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, []diag.Location{{File: "foobar"}}), nv) -} - -func TestFromTypedNilSliceRetainsLocation(t *testing.T) { - var src []string - ref := dyn.NewValue(nil, []diag.Location{{File: "foobar"}}) - - nv, err := FromTyped(src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, []diag.Location{{File: "foobar"}}), nv) -} - -func TestFromTypedForceSendFieldsComplexTypes(t *testing.T) { - type Inner struct { - Value string `json:"value"` - } - - tests := []struct { - name string - src any - }{ - { - name: "struct_pointer", - src: struct { - Field *Inner `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: nil, ForceSendFields: []string{"Field"}}, - }, - { - name: "struct_value", - src: struct { - Field Inner `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: Inner{}, ForceSendFields: []string{"Field"}}, - }, - { - name: "slice", - src: struct { - Field []string `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: nil, ForceSendFields: []string{"Field"}}, - }, - { - name: "map", - src: struct { - Field map[string]string `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: nil, ForceSendFields: []string{"Field"}}, - }, - { - name: "interface", - src: struct { - Field any `json:"field"` - ForceSendFields []string `json:"-"` - }{Field: (*string)(nil), ForceSendFields: []string{"Field"}}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - nv, err := FromTyped(tt.src, dyn.NilValue) - require.NoError(t, err) - - // All should include the field because it's in ForceSendFields - field := nv.Get("field") - assert.True(t, field.IsValid(), "field should be present due to ForceSendFields") - }) - } -} - -// Test embedded structs with ForceSendFields (separate test due to different structure) -func TestFromTypedForceSendFieldsEmbedded(t *testing.T) { - type Inner struct { - Field *string `json:"field"` - ForceSendFields []string `json:"-"` - } - type Outer struct { - Inner - Other string `json:"other"` - } - - src := Outer{ - Field: nil, - ForceSendFields: []string{"Field"}, - Other: "value", - } - - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - - // Both fields should be present - field := nv.Get("field") - other := nv.Get("other") - assert.True(t, field.IsValid(), "embedded field should be present due to ForceSendFields") - assert.Equal(t, dyn.KindNil, field.Kind(), "embedded field should be present due to ForceSendFields") - assert.Equal(t, dyn.V("value"), other) -} diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go deleted file mode 100644 index e41ed4a0a87..00000000000 --- a/libs/dyn/convert/normalize.go +++ /dev/null @@ -1,507 +0,0 @@ -package convert - -import ( - "fmt" - "reflect" - "slices" - "strconv" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" -) - -// NormalizeOption is the type for options that can be passed to Normalize. -type NormalizeOption int - -const ( - // IncludeMissingFields causes the normalization to include fields that defined on the given - // type but are missing in the source value. They are included with their zero values. - IncludeMissingFields NormalizeOption = iota - - // DropEmptyStrings drops struct fields whose value is an empty string, unless - // the field is tagged without omitempty (i.e. its zero value must be sent). - // This mirrors JSON serialization, where an omitempty string field set to "" - // is omitted from the request. Without this, an explicitly-set "" reaches - // ToTyped, which force-sends it, and the backend rejects it (e.g. - // "'' is not a valid cluster policy ID"). - DropEmptyStrings -) - -type normalizeOptions struct { - includeMissingFields bool - dropEmptyStrings bool -} - -func Normalize(dst any, src dyn.Value, opts ...NormalizeOption) (dyn.Value, diag.Diagnostics) { - var n normalizeOptions - for _, opt := range opts { - switch opt { - case IncludeMissingFields: - n.includeMissingFields = true - case DropEmptyStrings: - n.dropEmptyStrings = true - } - } - - return n.normalizeType(reflect.TypeOf(dst), src, []reflect.Type{}, dyn.EmptyPath) -} - -func (n normalizeOptions) normalizeType(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { - for typ.Kind() == reflect.Pointer { - typ = typ.Elem() - } - - switch typ.Kind() { - case reflect.Struct: - // Handle SDK native types as strings since they use custom JSON marshaling. - if slices.Contains(sdkNativeTypes, typ) { - return n.normalizeString(reflect.TypeFor[string](), src, path) - } - return n.normalizeStruct(typ, src, append(seen, typ), path) - case reflect.Map: - return n.normalizeMap(typ, src, append(seen, typ), path) - case reflect.Slice: - return n.normalizeSlice(typ, src, append(seen, typ), path) - case reflect.String: - return n.normalizeString(typ, src, path) - case reflect.Bool: - return n.normalizeBool(typ, src, path) - case reflect.Int, reflect.Int32, reflect.Int64: - return n.normalizeInt(typ, src, path) - case reflect.Float32, reflect.Float64: - return n.normalizeFloat(typ, src, path) - case reflect.Interface: - return n.normalizeInterface(typ, src, path) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diag.Errorf("unsupported type: %s", typ.Kind()) -} - -func nullWarning(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { - return diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("expected a %s value, found null", expected), - Locations: []diag.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - } -} - -func typeMismatch(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { - return diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()), - Locations: []diag.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - } -} - -// isAnchorContainer reports whether v is a YAML anchor or a non-empty -// sequence/map composed entirely of anchor containers. Anchors define reusable -// blocks and must not trigger "unknown field" warnings, including when nested -// inside a container. -func isAnchorContainer(v dyn.Value) bool { - if v.IsAnchor() { - return true - } - - var elements []dyn.Value - switch v.Kind() { - case dyn.KindSequence: - elements = v.MustSequence() - case dyn.KindMap: - elements = v.MustMap().Values() - default: - return false - } - - if len(elements) == 0 { - return false - } - for _, e := range elements { - if !isAnchorContainer(e) { - return false - } - } - return true -} - -func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindMap: - out := dyn.NewMapping() - info := getStructInfo(typ) - for _, pair := range src.MustMap().Pairs() { - pk := pair.Key - pv := pair.Value - - fieldName := pk.MustString() - index, ok := info.Fields[fieldName] - if !ok { - if !isAnchorContainer(pv) { - // Special case: provide a more helpful message for "valueFrom" vs "value_from" - if fieldName == "valueFrom" { - if _, hasValueFrom := info.Fields["value_from"]; hasValueFrom { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: "Use 'value_from' instead of 'valueFrom'", - Detail: "The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored.", - Locations: pk.Locations(), - Paths: dyn.ToStructPaths(path), - }) - continue - } - } - - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: "unknown field: " + fieldName, - Locations: pk.Locations(), - Paths: dyn.ToStructPaths(path), - }) - } - continue - } - - // Normalize the value according to the field type. - nv, err := n.normalizeType(typ.FieldByIndex(index).Type, pv, seen, path.Append(dyn.Key(pk.MustString()))) - if err != nil { - diags = diags.Extend(err) - // Skip the element if it cannot be normalized. - if !nv.IsValid() { - continue - } - } - - // Drop an empty string on an omitempty field so it is not force-sent - // to the backend (see DropEmptyStrings). ForceEmpty marks fields whose - // zero value must be kept, so those are left in place. - if n.dropEmptyStrings && !info.ForceEmpty[fieldName] { - if s, ok := nv.AsString(); ok && s == "" { - continue - } - } - - out.SetLoc(pk.MustString(), pk.Locations(), nv) - } - - // Return the normalized value if missing fields are not included. - if !n.includeMissingFields { - return dyn.NewValue(out, src.Locations()), diags - } - - // Populate missing fields with their zero values. - for k, index := range info.Fields { - if _, ok := out.GetByString(k); ok { - continue - } - - // Optionally dereference pointers to get the underlying field type. - ftyp := typ.FieldByIndex(index).Type - for ftyp.Kind() == reflect.Pointer { - ftyp = ftyp.Elem() - } - - // Skip field if we have already seen its type to avoid infinite recursion - // when filling in the zero value of a recursive type. - if slices.Contains(seen, ftyp) { - continue - } - - var v dyn.Value - switch ftyp.Kind() { - case reflect.Struct, reflect.Map: - v, _ = n.normalizeType(ftyp, dyn.V(map[string]dyn.Value{}), seen, path.Append(dyn.Key(k))) - case reflect.Slice: - v, _ = n.normalizeType(ftyp, dyn.V([]dyn.Value{}), seen, path.Append(dyn.Key(k))) - case reflect.String: - v, _ = n.normalizeType(ftyp, dyn.V(""), seen, path.Append(dyn.Key(k))) - case reflect.Bool: - v, _ = n.normalizeType(ftyp, dyn.V(false), seen, path.Append(dyn.Key(k))) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - v, _ = n.normalizeType(ftyp, dyn.V(int64(0)), seen, path.Append(dyn.Key(k))) - case reflect.Float32, reflect.Float64: - v, _ = n.normalizeType(ftyp, dyn.V(float64(0)), seen, path.Append(dyn.Key(k))) - default: - // Skip fields for which we do not have a natural [dyn.Value] equivalent. - // For example, we don't handle reflect.Complex* and reflect.Uint* types. - continue - } - if v.IsValid() { - out.SetLoc(k, nil, v) - } - } - - return dyn.NewValue(out, src.Locations()), diags - case dyn.KindNil: - return src, diags - - case dyn.KindString: - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - default: - // Fall through to the error case. - } - - // Cannot interpret as a struct. - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path)) -} - -func (n normalizeOptions) normalizeMap(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindMap: - out := dyn.NewMapping() - for _, pair := range src.MustMap().Pairs() { - pk := pair.Key - pv := pair.Value - - // Normalize the value according to the map element type. - nv, err := n.normalizeType(typ.Elem(), pv, seen, path.Append(dyn.Key(pk.MustString()))) - if err != nil { - diags = diags.Extend(err) - // Skip the element if it cannot be normalized. - if !nv.IsValid() { - continue - } - } - - out.SetLoc(pk.MustString(), pk.Locations(), nv) - } - - return dyn.NewValue(out, src.Locations()), diags - case dyn.KindNil: - return src, diags - - case dyn.KindString: - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - default: - // Fall through to the error case. - } - - // Cannot interpret as a map. - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path)) -} - -func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindSequence: - out := make([]dyn.Value, 0, len(src.MustSequence())) - for _, v := range src.MustSequence() { - // Normalize the value according to the slice element type. - v, err := n.normalizeType(typ.Elem(), v, seen, path.Append(dyn.Index(len(out)))) - if err != nil { - diags = diags.Extend(err) - // Skip the element if it cannot be normalized. - if !v.IsValid() { - continue - } - } - - out = append(out, v) - } - - return dyn.NewValue(out, src.Locations()), diags - case dyn.KindNil: - return src, diags - - case dyn.KindString: - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - default: - // Fall through to the error case. - } - - // Cannot interpret as a slice. - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindSequence, src, path)) -} - -func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindString: - return dyn.NewValue(src.MustString(), src.Locations()), nil - case dyn.KindBool: - return dyn.NewValue(strconv.FormatBool(src.MustBool()), src.Locations()), nil - case dyn.KindInt: - return dyn.NewValue(strconv.FormatInt(src.MustInt(), 10), src.Locations()), nil - case dyn.KindFloat: - return dyn.NewValue(strconv.FormatFloat(src.MustFloat(), 'f', -1, 64), src.Locations()), nil - case dyn.KindTime: - return dyn.NewValue(src.MustTime().String(), src.Locations()), nil - case dyn.KindNil: - // Return a warning if the field is present but has a null value. - return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindString, src, path)) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindString, src, path)) -} - -func (n normalizeOptions) normalizeBool(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindBool: - return dyn.NewValue(src.MustBool(), src.Locations()), nil - case dyn.KindString: - // See https://yaml.org/type/bool.html. - switch src.MustString() { - case "true", "True", "TRUE", "y", "Y", "yes", "Yes", "YES", "on", "On", "ON": - return dyn.NewValue(true, src.Locations()), nil - case "false", "False", "FALSE", "n", "N", "no", "No", "NO", "off", "Off", "OFF": - return dyn.NewValue(false, src.Locations()), nil - default: - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - - // Cannot interpret as a boolean. - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindBool, src, path)) - } - case dyn.KindNil: - // Return a warning if the field is present but has a null value. - return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindBool, src, path)) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindBool, src, path)) -} - -func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindInt: - return dyn.NewValue(src.MustInt(), src.Locations()), nil - case dyn.KindFloat: - out := int64(src.MustFloat()) - if src.MustFloat() != float64(out) { - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, src.MustFloat()), - Locations: []diag.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - }) - } - return dyn.NewValue(out, src.Locations()), nil - case dyn.KindString: - var err error - out, err := strconv.ParseInt(src.MustString(), 10, 64) - if err != nil { - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()), - Locations: []diag.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - }) - } - return dyn.NewValue(out, src.Locations()), nil - case dyn.KindNil: - // Return a warning if the field is present but has a null value. - return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindInt, src, path)) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindInt, src, path)) -} - -func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - - switch src.Kind() { - case dyn.KindFloat: - return dyn.NewValue(src.MustFloat(), src.Locations()), nil - case dyn.KindInt: - out := float64(src.MustInt()) - if src.MustInt() != int64(out) { - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, src.MustInt()), - Locations: []diag.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - }) - } - return dyn.NewValue(out, src.Locations()), nil - case dyn.KindString: - var err error - out, err := strconv.ParseFloat(src.MustString(), 64) - if err != nil { - // Return verbatim if it's a pure variable reference. - if dynvar.IsPureVariableReference(src.MustString()) { - return src, nil - } - - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()), - Locations: []diag.Location{src.Location()}, - Paths: dyn.ToStructPaths(path), - }) - } - return dyn.NewValue(out, src.Locations()), nil - case dyn.KindNil: - // Return a warning if the field is present but has a null value. - return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindFloat, src, path)) - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindFloat, src, path)) -} - -func (n normalizeOptions) normalizeInterface(_ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { - // Deal with every [dyn.Kind] here to ensure completeness. - switch src.Kind() { - case dyn.KindMap: - // Fall through - case dyn.KindSequence: - // Fall through - case dyn.KindString: - // Fall through - case dyn.KindBool: - // Fall through - case dyn.KindInt: - // Fall through - case dyn.KindFloat: - // Fall through - case dyn.KindTime: - // Conversion of a time value to an interface{}. - // The [dyn.Value.AsAny] equivalent for this kind is the [time.Time] struct. - // If we convert to a typed representation and back again, we cannot distinguish - // a [time.Time] struct from any other struct. - // - // Therefore, we normalize the time value to a string. - return dyn.NewValue(src.MustTime().String(), src.Locations()), nil - case dyn.KindNil: - // Fall through - default: - return dyn.InvalidValue, diag.Errorf("unsupported kind: %s", src.Kind()) - } - - return src, nil -} diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go deleted file mode 100644 index ac95aaafac6..00000000000 --- a/libs/dyn/convert/normalize_test.go +++ /dev/null @@ -1,1018 +0,0 @@ -package convert - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/databricks/cli/libs/structs/structpath" - "github.com/stretchr/testify/assert" -) - -func TestNormalizeStruct(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - vout, diags := Normalize(typ, vin) - assert.Empty(t, diags) - assert.Equal(t, vin, vout) -} - -func TestNormalizeStructElementDiagnostic(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V(map[string]dyn.Value{"an": dyn.V("error")}), - }) - - vout, diags := Normalize(typ, vin) - assert.Len(t, diags, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Locations: []diag.Location{{}}, - Paths: structpath.NewPathSlice("bar"), - }, diags[0]) - - // Elements that encounter an error during normalization are dropped. - assert.Equal(t, map[string]any{ - "foo": "bar", - }, vout.AsAny()) -} - -func TestNormalizeStructUnknownField(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - - m := dyn.NewMapping() - m.SetLoc("foo", nil, dyn.V("val-foo")) - - // Set the unknown field, with location information. - m.SetLoc("bar", []diag.Location{ - {File: "hello.yaml", Line: 1, Column: 1}, - {File: "world.yaml", Line: 2, Column: 2}, - }, dyn.V("var-bar")) - - vin := dyn.V(m) - - vout, diags := Normalize(typ, vin) - assert.Len(t, diags, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `unknown field: bar`, - // Assert location of the unknown field is included in the diagnostic. - Locations: []diag.Location{ - {File: "hello.yaml", Line: 1, Column: 1}, - {File: "world.yaml", Line: 2, Column: 2}, - }, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, diags[0]) - - // The field that can be mapped to the struct field is retained. - assert.Equal(t, map[string]any{ - "foo": "val-foo", - }, vout.AsAny()) -} - -func TestNormalizeStructNil(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.NilValue - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeStructError(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.V("string") - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Locations: []diag.Location{vin.Get("foo").Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeStructNestedError(t *testing.T) { - type Nested struct { - F1 int `json:"f1"` - F2 int `json:"f2"` - } - type Tmp struct { - Foo Nested `json:"foo"` - Bar Nested `json:"bar"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "f1": dyn.V("error"), - "f2": dyn.V(1), - }), - "bar": dyn.V(map[string]dyn.Value{ - "f1": dyn.V(1), - "f2": dyn.V("error"), - }), - }) - vout, err := Normalize(typ, vin) - assert.Len(t, err, 2) - - // Verify that valid fields are retained. - dynassert.Equal(t, - dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "f2": dyn.V(int64(1)), - }), - "bar": dyn.V(map[string]dyn.Value{ - "f1": dyn.V(int64(1)), - }), - }), - vout, - ) -} - -func TestNormalizeStructIncludeMissingFields(t *testing.T) { - type Nested struct { - String string `json:"string"` - } - - type Tmp struct { - // Verify that fields that are already set in the dynamic value are not overridden. - Existing string `json:"existing"` - - // Verify that structs are recursively normalized if not set. - Nested Nested `json:"nested"` - Ptr *Nested `json:"ptr"` - - // Verify that containers are also zero-initialized if not set. - Map map[string]string `json:"map"` - Slice []string `json:"slice"` - - // Verify that primitive types are zero-initialized if not set. - String string `json:"string"` - Bool bool `json:"bool"` - Int int `json:"int"` - Float float64 `json:"float"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "existing": dyn.V("already set"), - }) - vout, err := Normalize(typ, vin, IncludeMissingFields) - assert.Empty(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "existing": dyn.V("already set"), - "nested": dyn.V(map[string]dyn.Value{ - "string": dyn.V(""), - }), - "ptr": dyn.V(map[string]dyn.Value{ - "string": dyn.V(""), - }), - "map": dyn.V(map[string]dyn.Value{}), - "slice": dyn.V([]dyn.Value{}), - "string": dyn.V(""), - "bool": dyn.V(false), - "int": dyn.V(int64(0)), - "float": dyn.V(float64(0)), - }), vout) -} - -func TestNormalizeStructIncludeMissingFieldsOnRecursiveType(t *testing.T) { - type Tmp struct { - // Verify that structs are recursively normalized if not set. - Ptr *Tmp `json:"ptr"` - - // Verify that primitive types are zero-initialized if not set. - String string `json:"string"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "ptr": dyn.V(map[string]dyn.Value{ - "ptr": dyn.V(map[string]dyn.Value{ - "string": dyn.V("already set"), - }), - }), - }) - vout, err := Normalize(typ, vin, IncludeMissingFields) - assert.Empty(t, err) - dynassert.Equal(t, dyn.V(map[string]dyn.Value{ - "ptr": dyn.V(map[string]dyn.Value{ - "ptr": dyn.V(map[string]dyn.Value{ - // Note: the ptr field is not zero-initialized because that would recurse. - "string": dyn.V("already set"), - }), - "string": dyn.V(""), - }), - "string": dyn.V(""), - }), vout) -} - -func TestNormalizeStructVariableReference(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.NewValue("${var.foo}", []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeStructRandomStringError(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.NewValue("var foo", []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeStructIntError(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.NewValue(1, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found int`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeMap(t *testing.T) { - var typ map[string]string - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeMapElementDiagnostic(t *testing.T) { - var typ map[string]string - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V(map[string]dyn.Value{"an": dyn.V("error")}), - }) - - vout, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Locations: []diag.Location{{}}, - Paths: structpath.NewPathSlice("bar"), - }, err[0]) - - // Elements that encounter an error during normalization are dropped. - assert.Equal(t, map[string]any{ - "foo": "bar", - }, vout.AsAny()) -} - -func TestNormalizeMapNil(t *testing.T) { - var typ map[string]string - vin := dyn.NilValue - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeMapError(t *testing.T) { - var typ map[string]string - vin := dyn.V("string") - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeMapNestedError(t *testing.T) { - type Nested struct { - F1 int `json:"f1"` - F2 int `json:"f2"` - } - - var typ map[string]Nested - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "f1": dyn.V("error"), - "f2": dyn.V(1), - }), - "bar": dyn.V(map[string]dyn.Value{ - "f1": dyn.V(1), - "f2": dyn.V("error"), - }), - }) - vout, err := Normalize(typ, vin) - assert.Len(t, err, 2) - - // Verify that valid fields are retained. - dynassert.Equal(t, - dyn.V(map[string]dyn.Value{ - "foo": dyn.V(map[string]dyn.Value{ - "f2": dyn.V(int64(1)), - }), - "bar": dyn.V(map[string]dyn.Value{ - "f1": dyn.V(int64(1)), - }), - }), - vout, - ) -} - -func TestNormalizeMapVariableReference(t *testing.T) { - var typ map[string]string - vin := dyn.NewValue("${var.foo}", []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeMapRandomStringError(t *testing.T) { - var typ map[string]string - vin := dyn.NewValue("var foo", []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeMapIntError(t *testing.T) { - var typ map[string]string - vin := dyn.NewValue(1, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found int`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeSlice(t *testing.T) { - var typ []string - vin := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }) - - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeSliceElementDiagnostic(t *testing.T) { - var typ []string - vin := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - dyn.V(map[string]dyn.Value{"an": dyn.V("error")}), - }) - - vout, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Locations: []diag.Location{{}}, - Paths: []*structpath.PathNode{structpath.NewIndex(nil, 2)}, - }, err[0]) - - // Elements that encounter an error during normalization are dropped. - assert.Equal(t, []any{"foo", "bar"}, vout.AsAny()) -} - -func TestNormalizeSliceNil(t *testing.T) { - var typ []string - vin := dyn.NilValue - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeSliceError(t *testing.T) { - var typ []string - vin := dyn.V("string") - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found string`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeSliceNestedError(t *testing.T) { - type Nested struct { - F1 int `json:"f1"` - F2 int `json:"f2"` - } - - var typ []Nested - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "f1": dyn.V("error"), - "f2": dyn.V(1), - }), - dyn.V(map[string]dyn.Value{ - "f1": dyn.V(1), - "f2": dyn.V("error"), - }), - }) - vout, err := Normalize(typ, vin) - assert.Len(t, err, 2) - - // Verify that valid fields are retained. - dynassert.Equal(t, - dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "f2": dyn.V(int64(1)), - }), - dyn.V(map[string]dyn.Value{ - "f1": dyn.V(int64(1)), - }), - }), - vout, - ) -} - -func TestNormalizeSliceVariableReference(t *testing.T) { - var typ []string - vin := dyn.NewValue("${var.foo}", []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeSliceRandomStringError(t *testing.T) { - var typ []string - vin := dyn.NewValue("var foo", []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found string`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeSliceIntError(t *testing.T) { - var typ []string - vin := dyn.NewValue(1, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found int`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeString(t *testing.T) { - var typ string - vin := dyn.V("string") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeStringNil(t *testing.T) { - var typ string - vin := dyn.NewValue(nil, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a string value, found null`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeStringFromBool(t *testing.T) { - var typ string - vin := dyn.NewValue(true, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("true", vin.Locations()), vout) -} - -func TestNormalizeStringFromInt(t *testing.T) { - var typ string - vin := dyn.NewValue(123, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("123", vin.Locations()), vout) -} - -func TestNormalizeStringFromFloat(t *testing.T) { - var typ string - vin := dyn.NewValue(1.20, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("1.2", vin.Locations()), vout) -} - -func TestNormalizeStringFromTime(t *testing.T) { - var typ string - vin := dyn.NewValue(dyn.MustTime("2024-08-29"), []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("2024-08-29", vin.Locations()), vout) -} - -func TestNormalizeStringError(t *testing.T) { - var typ string - vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Locations: []diag.Location{{}}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeBool(t *testing.T) { - var typ bool - vin := dyn.V(true) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(true), vout) -} - -func TestNormalizeBoolNil(t *testing.T) { - var typ bool - vin := dyn.NewValue(nil, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a bool value, found null`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeBoolFromString(t *testing.T) { - var typ bool - - for _, c := range []struct { - Input string - Output bool - }{ - {"true", true}, - {"false", false}, - {"True", true}, - {"False", false}, - {"TRUE", true}, - {"FALSE", false}, - {"Y", true}, - {"N", false}, - {"on", true}, - {"off", false}, - } { - vin := dyn.V(c.Input) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(c.Output), vout) - } -} - -func TestNormalizeBoolFromStringVariableReference(t *testing.T) { - var typ bool - vin := dyn.V("${var.foo}") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeBoolFromStringError(t *testing.T) { - var typ bool - vin := dyn.V("abc") - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected bool, found string`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeBoolError(t *testing.T) { - var typ bool - vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected bool, found map`, - Locations: []diag.Location{{}}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeInt(t *testing.T) { - var typ int - vin := dyn.V(123) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(int64(123)), vout) -} - -func TestNormalizeIntNil(t *testing.T) { - var typ int - vin := dyn.NewValue(nil, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a int value, found null`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeIntFromFloat(t *testing.T) { - var typ int - vin := dyn.V(float64(1.0)) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(int64(1)), vout) -} - -func TestNormalizeIntFromFloatError(t *testing.T) { - var typ int - vin := dyn.V(1.5) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot accurately represent "1.5" as integer due to precision loss`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeIntFromString(t *testing.T) { - var typ int - vin := dyn.V("123") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(int64(123)), vout) -} - -func TestNormalizeIntFromStringVariableReference(t *testing.T) { - var typ int - vin := dyn.V("${var.foo}") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeIntFromStringError(t *testing.T) { - var typ int - vin := dyn.V("abc") - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot parse "abc" as an integer`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeIntError(t *testing.T) { - var typ int - vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected int, found map`, - Locations: []diag.Location{{}}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeFloat(t *testing.T) { - var typ float64 - vin := dyn.V(1.2) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(1.2), vout) -} - -func TestNormalizeFloatNil(t *testing.T) { - var typ float64 - vin := dyn.NewValue(nil, []diag.Location{{File: "file", Line: 1, Column: 1}}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a float value, found null`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeFloatFromInt(t *testing.T) { - var typ float64 - - // Maximum safe integer that can be accurately represented as a float. - vin := dyn.V(int64(9007199254740992)) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(float64(9007199254740992)), vout) -} - -func TestNormalizeFloatFromIntError(t *testing.T) { - var typ float64 - - // Minimum integer that cannot be accurately represented as a float. - vin := dyn.V(9007199254740992 + 1) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot accurately represent "9007199254740993" as floating point number due to precision loss`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeFloatFromString(t *testing.T) { - var typ float64 - vin := dyn.V("1.2") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.V(1.2), vout) -} - -func TestNormalizeFloatFromStringVariableReference(t *testing.T) { - var typ float64 - vin := dyn.V("${var.foo}") - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, vin, vout) -} - -func TestNormalizeFloatFromStringError(t *testing.T) { - var typ float64 - vin := dyn.V("abc") - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot parse "abc" as a floating point number`, - Locations: []diag.Location{vin.Location()}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeFloatError(t *testing.T) { - var typ float64 - vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) - _, err := Normalize(&typ, vin) - assert.Len(t, err, 1) - assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected float, found map`, - Locations: []diag.Location{{}}, - Paths: dyn.ToStructPaths(dyn.EmptyPath), - }, err[0]) -} - -func TestNormalizeAnchors(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "anchor": dyn.V("anchor").MarkAnchor(), - }) - - vout, err := Normalize(typ, vin) - assert.Empty(t, err) - - // The field that can be mapped to the struct field is retained. - assert.Equal(t, map[string]any{ - "foo": "bar", - }, vout.AsAny()) -} - -func TestNormalizeAnchorContainers(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - anchor := func() dyn.Value { - return dyn.V(map[string]dyn.Value{"name": dyn.V("x")}).MarkAnchor() - } - - tcases := []struct { - name string - value dyn.Value - wantWarn bool - }{ - { - name: "list of anchors", - value: dyn.V([]dyn.Value{anchor(), anchor()}), - wantWarn: false, - }, - { - name: "map of anchors", - value: dyn.V(map[string]dyn.Value{ - "a": anchor(), - "b": anchor(), - }), - wantWarn: false, - }, - { - name: "nested list of anchors", - value: dyn.V([]dyn.Value{dyn.V([]dyn.Value{anchor()})}), - wantWarn: false, - }, - { - name: "list with a non-anchor element", - value: dyn.V([]dyn.Value{anchor(), dyn.V(map[string]dyn.Value{"name": dyn.V("x")})}), - wantWarn: true, - }, - { - name: "empty list", - value: dyn.V([]dyn.Value{}), - wantWarn: true, - }, - { - name: "empty map", - value: dyn.V(map[string]dyn.Value{}), - wantWarn: true, - }, - } - - for _, tc := range tcases { - t.Run(tc.name, func(t *testing.T) { - var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "thing": tc.value, - }) - - vout, diags := Normalize(typ, vin) - if tc.wantWarn { - assert.Len(t, diags, 1) - assert.Equal(t, "unknown field: thing", diags[0].Summary) - } else { - assert.Empty(t, diags) - } - - // The unknown field is never retained regardless of the warning. - assert.Equal(t, map[string]any{ - "foo": "bar", - }, vout.AsAny()) - }) - } -} - -func TestNormalizeAnyFromSlice(t *testing.T) { - var typ any - v1 := dyn.NewValue(1, []diag.Location{{File: "file", Line: 1, Column: 1}}) - v2 := dyn.NewValue(2, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vin := dyn.NewValue([]dyn.Value{v1, v2}, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue([]dyn.Value{v1, v2}, []diag.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeAnyFromString(t *testing.T) { - var typ any - vin := dyn.NewValue("string", []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("string", []diag.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeAnyFromBool(t *testing.T) { - var typ any - vin := dyn.NewValue(false, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue(false, []diag.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeAnyFromInt(t *testing.T) { - var typ any - vin := dyn.NewValue(10, []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue(10, []diag.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeAnyFromTime(t *testing.T) { - var typ any - vin := dyn.NewValue(dyn.MustTime("2024-08-29"), []diag.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("2024-08-29", vin.Locations()), vout) -} - -func TestNormalizeStructDropEmptyStrings(t *testing.T) { - type Tmp struct { - // Optional: dropped when empty. - Opt string `json:"opt,omitempty"` - // Required (no omitempty): kept even when empty. - Req string `json:"req"` - } - - vin := dyn.V(map[string]dyn.Value{ - "opt": dyn.V(""), - "req": dyn.V(""), - }) - - vout, diags := Normalize(Tmp{}, vin, DropEmptyStrings) - assert.Empty(t, diags) - - _, hasOpt := vout.MustMap().Get(dyn.V("opt")) - assert.False(t, hasOpt, "empty omitempty string should be dropped") - req, hasReq := vout.MustMap().Get(dyn.V("req")) - assert.True(t, hasReq, "empty non-omitempty string should be kept") - assert.Empty(t, req.MustString()) -} - -func TestNormalizeStructDropEmptyStringsKeepsNonEmpty(t *testing.T) { - type Tmp struct { - Opt string `json:"opt,omitempty"` - } - - vin := dyn.V(map[string]dyn.Value{"opt": dyn.V("value")}) - - vout, diags := Normalize(Tmp{}, vin, DropEmptyStrings) - assert.Empty(t, diags) - opt, ok := vout.MustMap().Get(dyn.V("opt")) - assert.True(t, ok) - assert.Equal(t, "value", opt.MustString()) -} diff --git a/libs/dyn/convert/sdk_native_types.go b/libs/dyn/convert/sdk_native_types.go deleted file mode 100644 index b67d9b2b1e3..00000000000 --- a/libs/dyn/convert/sdk_native_types.go +++ /dev/null @@ -1,97 +0,0 @@ -package convert - -import ( - "encoding/json" - "fmt" - "reflect" - "slices" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" - sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" - sdkfieldmask "github.com/databricks/databricks-sdk-go/common/types/fieldmask" - sdktime "github.com/databricks/databricks-sdk-go/common/types/time" -) - -// sdkNativeTypes is a list of SDK native types that use custom JSON marshaling -// and should be treated as strings in dyn.Value. These types all implement -// json.Marshaler and json.Unmarshaler interfaces. -var sdkNativeTypes = []reflect.Type{ - reflect.TypeFor[sdkduration.Duration](), // Protobuf duration format (e.g., "300s") - reflect.TypeFor[sdktime.Time](), // RFC3339 timestamp format (e.g., "2023-12-25T10:30:00Z") - reflect.TypeFor[sdkfieldmask.FieldMask](), // Comma-separated paths (e.g., "name,age,email") -} - -// fromTypedSDKNative converts SDK native types to dyn.Value. -// SDK native types (duration.Duration, time.Time, fieldmask.FieldMask) use -// custom JSON marshaling with string representations. -func fromTypedSDKNative(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { - // Check that the reference value is compatible or nil. - switch ref.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(ref.MustString()) { - return ref, nil - } - case dyn.KindNil: - // Allow nil reference. - default: - return dyn.InvalidValue, fmt.Errorf("cannot convert SDK native type to dynamic type %#v", ref.Kind().String()) - } - - // Check for zero value first. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - - // Use JSON marshaling since SDK native types implement json.Marshaler. - jsonBytes, err := json.Marshal(src.Interface()) - if err != nil { - return dyn.InvalidValue, err - } - - // All SDK native types marshal to JSON strings. Unmarshal to get the raw string value. - // For example: duration.Duration(300s) -> JSON "300s" -> string "300s" - var str string - if err := json.Unmarshal(jsonBytes, &str); err != nil { - return dyn.InvalidValue, err - } - - // Handle empty string as zero value. - if str == "" && !slices.Contains(options, includeZeroValues) { - return dyn.NilValue, nil - } - - return dyn.V(str), nil -} - -// toTypedSDKNative converts a dyn.Value to an SDK native type. -// SDK native types (duration.Duration, time.Time, fieldmask.FieldMask) use -// custom JSON marshaling with string representations. -func toTypedSDKNative(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - // Use JSON unmarshaling since SDK native types implement json.Unmarshaler. - // Marshal the string to create a valid JSON string literal for unmarshaling. - jsonBytes, err := json.Marshal(src.MustString()) - if err != nil { - return err - } - return json.Unmarshal(jsonBytes, dst.Addr().Interface()) - case dyn.KindNil: - dst.SetZero() - return nil - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a string, found a %s", src.Kind()), - } -} diff --git a/libs/dyn/convert/sdk_native_types_test.go b/libs/dyn/convert/sdk_native_types_test.go deleted file mode 100644 index cd3d14fa129..00000000000 --- a/libs/dyn/convert/sdk_native_types_test.go +++ /dev/null @@ -1,417 +0,0 @@ -package convert - -import ( - "reflect" - "testing" - "time" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" - sdkfieldmask "github.com/databricks/databricks-sdk-go/common/types/fieldmask" - sdktime "github.com/databricks/databricks-sdk-go/common/types/time" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Roundtrip tests - verify SDK native types convert to dyn.Value and back for both value and pointer types - -func TestDurationRoundtrip(t *testing.T) { - tests := []struct { - name string - duration time.Duration - expectedString string - }{ - {"5min", 5 * time.Minute, "300s"}, - {"7days", 7 * 24 * time.Hour, "604800s"}, - {"1hour", 1 * time.Hour, "3600s"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Test value type - t.Run("value", func(t *testing.T) { - src := *sdkduration.New(tt.duration) - dynValue, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out sdkduration.Duration - err = ToTyped(&out, dynValue) - require.NoError(t, err) - assert.Equal(t, src.AsDuration(), out.AsDuration()) - }) - - // Test pointer type - t.Run("pointer", func(t *testing.T) { - src := *sdkduration.New(tt.duration) - dynValue, err := FromTyped(&src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out *sdkduration.Duration - err = ToTyped(&out, dynValue) - require.NoError(t, err) - require.NotNil(t, out) - assert.Equal(t, src.AsDuration(), out.AsDuration()) - }) - }) - } -} - -func TestTimeRoundtrip(t *testing.T) { - tests := []struct { - name string - time time.Time - expectedString string - }{ - { - "no_nanos", - time.Date(2023, 12, 25, 10, 30, 0, 0, time.UTC), - "2023-12-25T10:30:00Z", - }, - { - "with_nanos", - time.Date(2023, 12, 25, 10, 30, 0, 123456789, time.UTC), - "2023-12-25T10:30:00.123456789Z", - }, - { - "epoch", - time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC), - "1970-01-01T00:00:00Z", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Test value type - t.Run("value", func(t *testing.T) { - src := *sdktime.New(tt.time) - dynValue, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out sdktime.Time - err = ToTyped(&out, dynValue) - require.NoError(t, err) - assert.Equal(t, src.AsTime(), out.AsTime()) - }) - - // Test pointer type - t.Run("pointer", func(t *testing.T) { - src := *sdktime.New(tt.time) - dynValue, err := FromTyped(&src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out *sdktime.Time - err = ToTyped(&out, dynValue) - require.NoError(t, err) - require.NotNil(t, out) - assert.Equal(t, src.AsTime(), out.AsTime()) - }) - }) - } -} - -func TestFieldMaskRoundtrip(t *testing.T) { - tests := []struct { - name string - paths []string - expectedString string - }{ - {"single", []string{"name"}, "name"}, - {"multiple", []string{"name", "age", "email"}, "name,age,email"}, - {"nested", []string{"user.name", "user.email"}, "user.name,user.email"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Test value type - t.Run("value", func(t *testing.T) { - src := *sdkfieldmask.New(tt.paths) - dynValue, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out sdkfieldmask.FieldMask - err = ToTyped(&out, dynValue) - require.NoError(t, err) - assert.Equal(t, src.Paths, out.Paths) - }) - - // Test pointer type - t.Run("pointer", func(t *testing.T) { - src := *sdkfieldmask.New(tt.paths) - dynValue, err := FromTyped(&src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, tt.expectedString, dynValue.MustString()) - - var out *sdkfieldmask.FieldMask - err = ToTyped(&out, dynValue) - require.NoError(t, err) - require.NotNil(t, out) - assert.Equal(t, src.Paths, out.Paths) - }) - }) - } -} - -// Edge case tests - -func TestNilValues(t *testing.T) { - tests := []struct { - name string - typ any - }{ - {"duration", (*sdkduration.Duration)(nil)}, - {"time", (*sdktime.Time)(nil)}, - {"fieldmask", (*sdkfieldmask.FieldMask)(nil)}, - } - - t.Run("from_typed", func(t *testing.T) { - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - nv, err := FromTyped(tt.typ, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - }) - } - }) - - t.Run("normalize", func(t *testing.T) { - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - vout, diags := Normalize(tt.typ, dyn.NilValue) - assert.Len(t, diags, 1) - assert.Equal(t, diag.Warning, diags[0].Severity) - assert.Equal(t, `expected a string value, found null`, diags[0].Summary) - assert.Equal(t, dyn.InvalidValue, vout) - }) - } - }) -} - -func TestToTypedErrors(t *testing.T) { - wrongTypeInput := dyn.V(map[string]dyn.Value{"foo": dyn.V("bar")}) - - tests := []struct { - name string - destValue any - input dyn.Value - expectedError string - }{ - { - name: "duration_wrong_type", - destValue: new(sdkduration.Duration), - input: wrongTypeInput, - expectedError: "expected a string", - }, - { - name: "duration_invalid_format", - destValue: new(sdkduration.Duration), - input: dyn.V("7d"), - expectedError: "invalid google.protobuf.Duration value", - }, - { - name: "time_wrong_type", - destValue: new(sdktime.Time), - input: wrongTypeInput, - expectedError: "expected a string", - }, - { - name: "time_invalid_format", - destValue: new(sdktime.Time), - input: dyn.V("not-a-time"), - expectedError: "invalid google.protobuf.Timestamp value", - }, - { - name: "fieldmask_wrong_type", - destValue: new(sdkfieldmask.FieldMask), - input: wrongTypeInput, - expectedError: "expected a string", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := ToTyped(tt.destValue, tt.input) - require.Error(t, err) - assert.Contains(t, err.Error(), tt.expectedError) - }) - } -} - -func TestFromTypedVariableReferences(t *testing.T) { - tests := []struct { - name string - src any - }{ - { - name: "duration", - src: *sdkduration.New(5 * time.Minute), - }, - { - name: "time", - src: *sdktime.New(time.Date(2023, 12, 25, 10, 30, 0, 0, time.UTC)), - }, - { - name: "fieldmask", - src: *sdkfieldmask.New([]string{"name", "age"}), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ref := dyn.V("${var.foo}") - nv, err := FromTyped(tt.src, ref) - require.NoError(t, err) - assert.Equal(t, dyn.V("${var.foo}"), nv) - }) - } -} - -func TestToTypedVariableReferences(t *testing.T) { - tests := []struct { - name string - dst any - zero any - }{ - { - name: "duration", - dst: sdkduration.New(5 * time.Minute), - zero: sdkduration.Duration{}, - }, - { - name: "time", - dst: sdktime.New(time.Date(2023, 12, 25, 10, 30, 0, 0, time.UTC)), - zero: sdktime.Time{}, - }, - { - name: "fieldmask", - dst: sdkfieldmask.New([]string{"name", "age"}), - zero: sdkfieldmask.FieldMask{}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := ToTyped(tt.dst, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.Equal(t, tt.zero, reflect.ValueOf(tt.dst).Elem().Interface()) - }) - } -} - -func TestSpecialCases(t *testing.T) { - t.Run("duration_zero", func(t *testing.T) { - var src sdkduration.Duration - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - }) - - t.Run("fieldmask_empty_fromtyped", func(t *testing.T) { - src := sdkfieldmask.New([]string{}) - nv, err := FromTyped(src, dyn.NilValue) - require.NoError(t, err) - // Empty field mask marshals to empty string - assert.Equal(t, dyn.V(""), nv) - }) - - t.Run("fieldmask_empty_totyped", func(t *testing.T) { - var out sdkfieldmask.FieldMask - err := ToTyped(&out, dyn.V("")) - require.NoError(t, err) - assert.Empty(t, out.Paths) - }) -} - -// End-to-end tests with structs containing SDK native types - -func TestSDKTypesRoundTripWithPostgresBranchSpec(t *testing.T) { - type BranchSpec struct { - ExpireTime *sdktime.Time `json:"expire_time,omitempty"` - SourceBranchTime *sdktime.Time `json:"source_branch_time,omitempty"` - Ttl *sdkduration.Duration `json:"ttl,omitempty"` - IsProtected bool `json:"is_protected,omitempty"` - } - - original := BranchSpec{ - ExpireTime: sdktime.New(time.Date(2024, 12, 31, 23, 59, 59, 0, time.UTC)), - SourceBranchTime: sdktime.New(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)), - Ttl: sdkduration.New(7 * 24 * time.Hour), - IsProtected: true, - } - - dynValue, err := FromTyped(original, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, "2024-12-31T23:59:59Z", dynValue.Get("expire_time").MustString()) - assert.Equal(t, "2024-01-01T00:00:00Z", dynValue.Get("source_branch_time").MustString()) - assert.Equal(t, "604800s", dynValue.Get("ttl").MustString()) - assert.True(t, dynValue.Get("is_protected").MustBool()) - - var roundtrip BranchSpec - err = ToTyped(&roundtrip, dynValue) - require.NoError(t, err) - require.NotNil(t, roundtrip.ExpireTime) - require.NotNil(t, roundtrip.SourceBranchTime) - require.NotNil(t, roundtrip.Ttl) - assert.Equal(t, original.ExpireTime.AsTime(), roundtrip.ExpireTime.AsTime()) - assert.Equal(t, original.SourceBranchTime.AsTime(), roundtrip.SourceBranchTime.AsTime()) - assert.Equal(t, original.Ttl.AsDuration(), roundtrip.Ttl.AsDuration()) - assert.Equal(t, original.IsProtected, roundtrip.IsProtected) -} - -func TestSDKTypesRoundTripWithUpdateRequest(t *testing.T) { - type UpdateRequest struct { - Name string `json:"name"` - UpdateMask sdkfieldmask.FieldMask `json:"update_mask"` - } - - original := UpdateRequest{ - Name: "projects/123/branches/456", - UpdateMask: *sdkfieldmask.New([]string{"spec.ttl", "spec.is_protected"}), - } - - dynValue, err := FromTyped(original, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, "projects/123/branches/456", dynValue.Get("name").MustString()) - assert.Equal(t, "spec.ttl,spec.is_protected", dynValue.Get("update_mask").MustString()) - - var roundtrip UpdateRequest - err = ToTyped(&roundtrip, dynValue) - require.NoError(t, err) - assert.Equal(t, original.Name, roundtrip.Name) - assert.Equal(t, []string{"spec.ttl", "spec.is_protected"}, roundtrip.UpdateMask.Paths) -} - -func TestSDKTypesNormalizeWithStruct(t *testing.T) { - type BranchSpec struct { - ExpireTime *sdktime.Time `json:"expire_time,omitempty"` - Ttl *sdkduration.Duration `json:"ttl,omitempty"` - IsProtected bool `json:"is_protected,omitempty"` - } - - var typ BranchSpec - vin := dyn.V(map[string]dyn.Value{ - "expire_time": dyn.V("2024-12-31T23:59:59Z"), - "ttl": dyn.V("604800s"), - "is_protected": dyn.V(true), - }) - - vout, diags := Normalize(typ, vin) - assert.Empty(t, diags) - assert.Equal(t, "2024-12-31T23:59:59Z", vout.Get("expire_time").MustString()) - assert.Equal(t, "604800s", vout.Get("ttl").MustString()) - assert.True(t, vout.Get("is_protected").MustBool()) - - var out BranchSpec - err := ToTyped(&out, vout) - require.NoError(t, err) - require.NotNil(t, out.ExpireTime) - require.NotNil(t, out.Ttl) - assert.Equal(t, time.Date(2024, 12, 31, 23, 59, 59, 0, time.UTC), out.ExpireTime.AsTime()) - assert.Equal(t, 7*24*time.Hour, out.Ttl.AsDuration()) - assert.True(t, out.IsProtected) -} diff --git a/libs/dyn/convert/struct_info.go b/libs/dyn/convert/struct_info.go deleted file mode 100644 index 69e8f868ed5..00000000000 --- a/libs/dyn/convert/struct_info.go +++ /dev/null @@ -1,211 +0,0 @@ -package convert - -import ( - "reflect" - "slices" - "sync" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/structs/structtag" -) - -// structInfo holds the type information we need to efficiently -// convert data from a [dyn.Value] to a Go struct. -type structInfo struct { - // FieldNames is ordered list of fields - FieldNames []string - - // Fields maps the JSON-name of the field to the field's index for use with [FieldByIndex]. - Fields map[string][]int - - // ValueField maps to the field with a [dyn.Value]. - // The underlying type is expected to only have one of these. - ValueField []int - - // Tracks which fields do not have omitempty annotation - ForceEmpty map[string]bool - - // Maps JSON-name of the field to Golang struct name - GolangNames map[string]string - - // ForceSendFieldsIndex maps the JSON-name of the field to the index path (for - // use with [reflect.Value.FieldByIndex]) of the ForceSendFields slice that - // governs it: the one declared by the struct that also declares the field. - // The path is static per type, so we resolve it once here rather than walking - // the value at conversion time. It can be more than one element deep because a - // field's declaring struct may be embedded several levels down (e.g. - // PostgresProject -> PostgresProjectConfig -> ProjectSpec). A field whose - // declaring struct has no ForceSendFields has no entry. - ForceSendFieldsIndex map[string][]int -} - -// structInfoCache caches type information. -var structInfoCache = make(map[reflect.Type]structInfo) - -// structInfoCacheLock guards concurrent access to structInfoCache. -var structInfoCacheLock sync.Mutex - -// getStructInfo returns the [structInfo] for the given type. -// It lazily populates a cache, so the first call for a given -// type is slower than subsequent calls for that same type. -func getStructInfo(typ reflect.Type) structInfo { - structInfoCacheLock.Lock() - defer structInfoCacheLock.Unlock() - - si, ok := structInfoCache[typ] - if !ok { - si = buildStructInfo(typ) - structInfoCache[typ] = si - } - - return si -} - -// buildStructInfo populates a new [structInfo] for the given type. -func buildStructInfo(typ reflect.Type) structInfo { - out := structInfo{ - Fields: make(map[string][]int), - ForceEmpty: make(map[string]bool), - GolangNames: make(map[string]string), - ForceSendFieldsIndex: make(map[string][]int), - } - - // Queue holds the indexes of the structs to visit. - // It is initialized with a single empty slice to visit the top level struct. - queue := [][]int{{}} - for i := 0; i < len(queue); i++ { - prefix := queue[i] - - // Traverse embedded anonymous types (if prefix is non-empty). - styp := typ - if len(prefix) > 0 { - styp = styp.FieldByIndex(prefix).Type - } - - // Dereference pointer type. - if styp.Kind() == reflect.Pointer { - styp = styp.Elem() - } - - // Index path to the ForceSendFields declared by this struct, if any. All - // fields declared directly by this struct are governed by it. The len==1 - // check excludes a ForceSendFields promoted from an embedded struct: that - // one governs the embedded struct's own fields, which we visit separately. - var forceSendFieldsIndex []int - if sf, ok := styp.FieldByName("ForceSendFields"); ok && len(sf.Index) == 1 { - forceSendFieldsIndex = append(slices.Clone(prefix), sf.Index...) - } - - nf := styp.NumField() - for j := range nf { - sf := styp.Field(j) - - // Recurse into anonymous fields. - if sf.Anonymous { - queue = append(queue, append(prefix, sf.Index...)) - continue - } - - // If this field has type [dyn.Value], we populate it with the source [dyn.Value] from [ToTyped]. - if sf.IsExported() && sf.Type == configValueType { - if out.ValueField != nil { - panic("multiple dyn.Value fields") - } - out.ValueField = append(prefix, sf.Index...) - continue - } - - jtag := structtag.JSONTag(sf.Tag.Get("json")) - name := jtag.Name() - if name == "" || name == "-" { - continue - } - - // Top level fields always take precedence. - // Therefore, if it is already set, we ignore it. - if _, ok := out.Fields[name]; ok { - continue - } - - out.FieldNames = append(out.FieldNames, name) - out.Fields[name] = append(prefix, sf.Index...) - if !jtag.OmitEmpty() && !jtag.OmitZero() { - out.ForceEmpty[name] = true - } - out.GolangNames[name] = sf.Name - - // The field is declared directly in this struct, so it is governed by - // this struct's ForceSendFields (if it has one). - if forceSendFieldsIndex != nil { - out.ForceSendFieldsIndex[name] = forceSendFieldsIndex - } - } - } - - return out -} - -type FieldValue struct { - Key string - Value reflect.Value - IsForced bool -} - -func (s *structInfo) FieldValues(v reflect.Value) []FieldValue { - out := make([]FieldValue, 0, len(s.Fields)) - - for _, k := range s.FieldNames { - fv := fieldByIndex(v, s.Fields[k]) - - if fv.IsValid() { - isForced := true - - // TODO: we should use isEmptyForOmitEmpty instead of IsZero() - if fv.IsZero() { - isForced = s.isForceSend(v, k) - } - - out = append(out, FieldValue{ - Key: k, - Value: fv, - IsForced: isForced, - }) - } - } - - return out -} - -// isForceSend reports whether the field named k is listed in the ForceSendFields -// that governs it (see structInfo.ForceSendFieldsIndex). -func (s *structInfo) isForceSend(v reflect.Value, k string) bool { - index, ok := s.ForceSendFieldsIndex[k] - if !ok { - return false - } - fsf := fieldByIndex(v, index) - if !fsf.IsValid() { - return false - } - return slices.Contains(fsf.Interface().([]string), s.GolangNames[k]) -} - -// fieldByIndex resolves the value at the given index path, dereferencing embedded -// pointer structs on the way. It returns an invalid value if a nil pointer is met. -func fieldByIndex(v reflect.Value, index []int) reflect.Value { - for i, x := range index { - if i > 0 { - if v.Kind() == reflect.Pointer && v.Type().Elem().Kind() == reflect.Struct { - if v.IsNil() { - return reflect.Value{} - } - v = v.Elem() - } - } - v = v.Field(x) - } - return v -} - -// Type of [dyn.Value]. -var configValueType = reflect.TypeFor[dyn.Value]() diff --git a/libs/dyn/convert/struct_info_test.go b/libs/dyn/convert/struct_info_test.go deleted file mode 100644 index f921523c391..00000000000 --- a/libs/dyn/convert/struct_info_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package convert - -import ( - "reflect" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestStructInfoPlain(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - - // Baz must be skipped. - Baz string `json:""` - - // Qux must be skipped. - Qux string `json:"-"` - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Len(t, si.Fields, 2) - assert.Equal(t, []int{0}, si.Fields["foo"]) - assert.Equal(t, []int{1}, si.Fields["bar"]) -} - -func TestStructInfoAnonymousByValue(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - Bar - } - - type Tmp struct { - Foo - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Len(t, si.Fields, 2) - assert.Equal(t, []int{0, 0}, si.Fields["foo"]) - assert.Equal(t, []int{0, 1, 0}, si.Fields["bar"]) -} - -func TestStructInfoAnonymousByValuePrecedence(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - Bar - } - - type Tmp struct { - // "foo" comes from [Foo]. - Foo - // "bar" comes from [Bar] directly, not through [Foo]. - Bar - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Len(t, si.Fields, 2) - assert.Equal(t, []int{0, 0}, si.Fields["foo"]) - assert.Equal(t, []int{1, 0}, si.Fields["bar"]) -} - -func TestStructInfoAnonymousByPointer(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - *Bar - } - - type Tmp struct { - *Foo - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Len(t, si.Fields, 2) - assert.Equal(t, []int{0, 0}, si.Fields["foo"]) - assert.Equal(t, []int{0, 1, 0}, si.Fields["bar"]) -} - -func TestStructInfoFieldValues(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar"` - } - - src := Tmp{ - Foo: "foo", - Bar: "bar", - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Len(t, fv, 2) - assert.Equal(t, "foo", fv[0].Key) - assert.True(t, reflect.ValueOf("foo").Equal(fv[0].Value)) - assert.Equal(t, "bar", fv[1].Key) - assert.True(t, reflect.ValueOf("bar").Equal(fv[1].Value)) -} - -func TestStructInfoFieldValuesAnonymousByValue(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - Bar - } - - type Tmp struct { - Foo - } - - src := Tmp{ - Foo: Foo{ - Foo: "foo", - Bar: Bar{ - Bar: "bar", - }, - }, - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Len(t, fv, 2) - assert.Equal(t, "foo", fv[0].Key) - assert.Equal(t, "bar", fv[1].Key) -} - -func TestStructInfoFieldValuesAnonymousByPointer(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - *Bar - } - - type Tmp struct { - *Foo - } - - // Test that the embedded fields are dereferenced properly. - t.Run("all are set", func(t *testing.T) { - src := Tmp{ - Foo: &Foo{ - Foo: "foo", - Bar: &Bar{ - Bar: "bar", - }, - }, - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Len(t, fv, 2) - assert.Equal(t, "foo", fv[0].Key) - assert.Equal(t, "bar", fv[1].Key) - }) - - // Test that fields of embedded types are skipped if the embedded type is nil. - t.Run("top level is set", func(t *testing.T) { - src := Tmp{ - Foo: &Foo{ - Foo: "foo", - Bar: nil, - }, - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Len(t, fv, 1) - assert.Equal(t, "foo", fv[0].Key) - }) - - // Test that fields of embedded types are skipped if the embedded type is nil. - t.Run("none are set", func(t *testing.T) { - src := Tmp{ - Foo: nil, - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - fv := si.FieldValues(reflect.ValueOf(src)) - assert.Empty(t, fv) - }) -} - -func TestStructInfoValueFieldAbsent(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.Nil(t, si.ValueField) -} - -func TestStructInfoValueFieldPresent(t *testing.T) { - type Tmp struct { - Foo dyn.Value - } - - si := getStructInfo(reflect.TypeFor[Tmp]()) - assert.NotNil(t, si.ValueField) -} - -func TestStructInfoValueFieldMultiple(t *testing.T) { - type Tmp struct { - Foo dyn.Value - Bar dyn.Value - } - - assert.Panics(t, func() { - getStructInfo(reflect.TypeFor[Tmp]()) - }) -} diff --git a/libs/dyn/convert/to_typed.go b/libs/dyn/convert/to_typed.go deleted file mode 100644 index 87b0fe7d8ca..00000000000 --- a/libs/dyn/convert/to_typed.go +++ /dev/null @@ -1,350 +0,0 @@ -package convert - -import ( - "fmt" - "reflect" - "slices" - "strconv" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" -) - -// Populate a destination typed value from a source dynamic value. -// -// At any point while walking the destination type tree using -// reflection, if this function sees an exported field with type dyn.Value it -// will populate that field with the appropriate source dynamic value. -// see PR: https://github.com/databricks/cli/pull/1010 -func ToTyped(dst any, src dyn.Value) error { - dstv := reflect.ValueOf(dst) - - // Dereference pointer if necessary - for dstv.Kind() == reflect.Pointer { - // If the source value is nil and the destination is a settable pointer, - // set the destination to nil. Also see `end_to_end_test.go`. - if dstv.CanSet() && src.Kind() == dyn.KindNil { - dstv.SetZero() - return nil - } - if dstv.IsNil() { - dstv.Set(reflect.New(dstv.Type().Elem())) - } - dstv = dstv.Elem() - } - - // Verify that vv is settable. - if !dstv.CanSet() { - panic("cannot set destination value") - } - - switch dstv.Kind() { - case reflect.Struct: - // Handle SDK native types using JSON unmarshaling. - if slices.Contains(sdkNativeTypes, dstv.Type()) { - return toTypedSDKNative(dstv, src) - } - return toTypedStruct(dstv, src) - case reflect.Map: - return toTypedMap(dstv, src) - case reflect.Slice: - return toTypedSlice(dstv, src) - case reflect.String: - return toTypedString(dstv, src) - case reflect.Bool: - return toTypedBool(dstv, src) - case reflect.Int, reflect.Int32, reflect.Int64: - return toTypedInt(dstv, src) - case reflect.Float32, reflect.Float64: - return toTypedFloat(dstv, src) - case reflect.Interface: - return toTypedInterface(dstv, src) - default: - // Fall through to the error case. - } - - return fmt.Errorf("unsupported type: %s", dstv.Kind()) -} - -func toTypedStruct(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindMap: - // Zero the destination struct such that fields - // that aren't present in [src] are cleared. - dst.SetZero() - - info := getStructInfo(dst.Type()) - - for _, pair := range src.MustMap().Pairs() { - pk := pair.Key - pv := pair.Value - jsonKey := pk.MustString() - - index, ok := info.Fields[jsonKey] - if !ok { - // Ignore unknown fields. - // A warning will be printed later. See PR #904. - continue - } - - f := getOrNewFieldByIndex(dst, index) - - err := ToTyped(f.Addr().Interface(), pv) - if err != nil { - return err - } - - if pv.IsZero() { - // The field's zero value must still serialize, so append its Go name - // to the ForceSendFields of the struct that declares it. That struct - // shares a prefix with the field's index path, so it is already - // allocated by the walk above. - if fsfIndex, ok := info.ForceSendFieldsIndex[jsonKey]; ok { - fsf := getOrNewFieldByIndex(dst, fsfIndex) - fsf.Set(reflect.Append(fsf, reflect.ValueOf(info.GolangNames[jsonKey]))) - } - } - } - - // Populate field(s) for [dyn.Value], if any. - if info.ValueField != nil { - vv := dst.FieldByIndex(info.ValueField) - vv.Set(reflect.ValueOf(src)) - } - - return nil - case dyn.KindNil: - dst.SetZero() - return nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a map, found a %s", src.Kind()), - } -} - -// getOrNewFieldByIndex resolves the value at the given index path within an addressable -// struct, allocating intermediate structs embedded as pointer types along the way. -// Code inspired by [reflect.FieldByIndex] implementation. -func getOrNewFieldByIndex(v reflect.Value, index []int) reflect.Value { - for i, x := range index { - if i > 0 { - if v.Kind() == reflect.Pointer { - if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) - } - v = v.Elem() - } - } - v = v.Field(x) - } - return v -} - -func toTypedMap(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindMap: - m := src.MustMap() - - // Always overwrite. - dst.Set(reflect.MakeMapWithSize(dst.Type(), m.Len())) - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - kv := reflect.ValueOf(pk.MustString()) - kt := dst.Type().Key() - vv := reflect.New(dst.Type().Elem()) - err := ToTyped(vv.Interface(), pv) - if err != nil { - return err - } - dst.SetMapIndex(kv.Convert(kt), vv.Elem()) - } - return nil - case dyn.KindNil: - dst.SetZero() - return nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a map, found a %s", src.Kind()), - } -} - -func toTypedSlice(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindSequence: - seq := src.MustSequence() - - // Always overwrite. - dst.Set(reflect.MakeSlice(dst.Type(), len(seq), len(seq))) - for i := range seq { - err := ToTyped(dst.Index(i).Addr().Interface(), seq[i]) - if err != nil { - return err - } - } - return nil - case dyn.KindNil: - dst.SetZero() - return nil - case dyn.KindString: - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a sequence, found a %s", src.Kind()), - } -} - -func toTypedString(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindString: - dst.SetString(src.MustString()) - return nil - case dyn.KindBool: - dst.SetString(strconv.FormatBool(src.MustBool())) - return nil - case dyn.KindInt: - dst.SetString(strconv.FormatInt(src.MustInt(), 10)) - return nil - case dyn.KindFloat: - dst.SetString(strconv.FormatFloat(src.MustFloat(), 'f', -1, 64)) - return nil - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a string, found a %s", src.Kind()), - } -} - -func toTypedBool(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindBool: - dst.SetBool(src.MustBool()) - return nil - case dyn.KindString: - // See https://yaml.org/type/bool.html. - switch src.MustString() { - case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON", "true", "True", "TRUE": - dst.SetBool(true) - return nil - case "n", "N", "no", "No", "NO", "off", "Off", "OFF", "false", "False", "FALSE": - dst.SetBool(false) - return nil - } - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a boolean, found a %s", src.Kind()), - } -} - -func toTypedInt(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindInt: - dst.SetInt(src.MustInt()) - return nil - case dyn.KindFloat: - v := src.MustFloat() - if v == float64(int64(v)) { - // If the destination is smaller than int64, but the value to set is bigger - // then destination overflows and is set to -1 - dst.SetInt(int64(src.MustFloat())) - return nil - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected an int, found a %s", src.Kind()), - } - case dyn.KindString: - if i64, err := strconv.ParseInt(src.MustString(), 10, 64); err == nil { - dst.SetInt(i64) - return nil - } - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected an int, found a %s", src.Kind()), - } -} - -func toTypedFloat(dst reflect.Value, src dyn.Value) error { - switch src.Kind() { - case dyn.KindFloat: - dst.SetFloat(src.MustFloat()) - return nil - case dyn.KindString: - if f64, err := strconv.ParseFloat(src.MustString(), 64); err == nil { - dst.SetFloat(f64) - return nil - } - // Ignore pure variable references (e.g. ${var.foo}). - if dynvar.IsPureVariableReference(src.MustString()) { - dst.SetZero() - return nil - } - default: - // Fall through to the error case. - } - - return TypeError{ - value: src, - msg: fmt.Sprintf("expected a float, found a %s", src.Kind()), - } -} - -func toTypedInterface(dst reflect.Value, src dyn.Value) error { - if src.Kind() == dyn.KindNil { - dst.Set(reflect.Zero(dst.Type())) - return nil - } - - dst.Set(reflect.ValueOf(src.AsAny())) - return nil -} diff --git a/libs/dyn/convert/to_typed_test.go b/libs/dyn/convert/to_typed_test.go deleted file mode 100644 index a838dfab33f..00000000000 --- a/libs/dyn/convert/to_typed_test.go +++ /dev/null @@ -1,790 +0,0 @@ -package convert - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestToTypedStruct(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - - // Baz must be skipped. - Baz string `json:""` - - // Qux must be skipped. - Qux string `json:"-"` - } - - var out Tmp - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo) - assert.Equal(t, "baz", out.Bar) -} - -func TestToTypedStructOverwrite(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - - // Baz must be skipped. - Baz string `json:""` - - // Qux must be skipped. - Qux string `json:"-"` - } - - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo) - assert.Equal(t, "baz", out.Bar) -} - -func TestToTypedStructClearFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - } - - // Struct value with non-empty fields. - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - - // Value is an empty map. - v := dyn.V(map[string]dyn.Value{}) - - // The previously set fields should be cleared. - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructClearFieldsForceSend(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - ForceSendFields []string `json:"-"` - } - - // Struct value with non-empty fields. - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - - // Value is an empty map. - v := dyn.V(map[string]dyn.Value{}) - - // The previously set fields should be cleared. - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructZeroFields(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - } - - // Struct value with non-empty fields. - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - - // Value is an empty map. - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(""), - "bar": dyn.V(""), - }) - - // The previously set fields should be cleared. - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructZeroFieldsForceSend(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - Bar string `json:"bar,omitempty"` - NumWorkers int `json:"num_workers,omitempty"` - ForceSendFields []string `json:"-"` - } - - // Struct value with non-empty fields. - out := Tmp{ - Foo: "baz", - Bar: "qux", - } - - // Value is an empty map. - m := dyn.Mapping{} - m.SetLoc("foo", nil, dyn.V("")) - m.SetLoc("bar", nil, dyn.V("")) - m.SetLoc("num_workers", nil, dyn.V(int64(0))) - v := dyn.V(m) - - // The previously set fields should be cleared. - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, Tmp{ForceSendFields: []string{"Foo", "Bar", "NumWorkers"}}, out) -} - -func TestToTypedStructAnonymousByValue(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - Bar - } - - type Tmp struct { - Foo - } - - var out Tmp - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo.Foo) - assert.Equal(t, "baz", out.Bar.Bar) -} - -func TestToTypedStructAnonymousByPointer(t *testing.T) { - type Bar struct { - Bar string `json:"bar"` - } - - type Foo struct { - Foo string `json:"foo"` - *Bar - } - - type Tmp struct { - *Foo - } - - var out Tmp - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo.Foo) - assert.Equal(t, "baz", out.Bar.Bar) -} - -func TestToTypedStructNil(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - out := Tmp{} - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructNilOverwrite(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - } - - out := Tmp{"bar"} - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Equal(t, Tmp{}, out) -} - -func TestToTypedStructWithValueField(t *testing.T) { - type Tmp struct { - Foo string `json:"foo"` - - ConfigValue dyn.Value - } - - var out Tmp - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, "bar", out.Foo) - assert.Equal(t, v, out.ConfigValue) -} - -func TestToTypedMap(t *testing.T) { - out := map[string]string{} - - v := dyn.V(map[string]dyn.Value{ - "key": dyn.V("value"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 1) - assert.Equal(t, "value", out["key"]) -} - -func TestToTypedMapOverwrite(t *testing.T) { - out := map[string]string{ - "foo": "bar", - } - - v := dyn.V(map[string]dyn.Value{ - "bar": dyn.V("qux"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 1) - assert.Equal(t, "qux", out["bar"]) -} - -func TestToTypedMapWithPointerElement(t *testing.T) { - var out map[string]*string - - v := dyn.V(map[string]dyn.Value{ - "key": dyn.V("value"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 1) - assert.Equal(t, "value", *out["key"]) -} - -func TestToTypedMapNil(t *testing.T) { - out := map[string]string{} - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedMapNilOverwrite(t *testing.T) { - out := map[string]string{ - "foo": "bar", - } - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedSlice(t *testing.T) { - var out []string - - v := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 2) - assert.Equal(t, "foo", out[0]) - assert.Equal(t, "bar", out[1]) -} - -func TestToTypedSliceOverwrite(t *testing.T) { - out := []string{"qux"} - - v := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 2) - assert.Equal(t, "foo", out[0]) - assert.Equal(t, "bar", out[1]) -} - -func TestToTypedSliceWithPointerElement(t *testing.T) { - var out []*string - - v := dyn.V([]dyn.Value{ - dyn.V("foo"), - dyn.V("bar"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 2) - assert.Equal(t, "foo", *out[0]) - assert.Equal(t, "bar", *out[1]) -} - -func TestToTypedSliceNil(t *testing.T) { - var out []string - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedSliceNilOverwrite(t *testing.T) { - out := []string{"foo"} - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedString(t *testing.T) { - var out string - err := ToTyped(&out, dyn.V("foo")) - require.NoError(t, err) - assert.Equal(t, "foo", out) -} - -func TestToTypedStringOverwrite(t *testing.T) { - out := "bar" - err := ToTyped(&out, dyn.V("foo")) - require.NoError(t, err) - assert.Equal(t, "foo", out) -} - -func TestToTypedStringFromBool(t *testing.T) { - var out string - err := ToTyped(&out, dyn.V(true)) - require.NoError(t, err) - assert.Equal(t, "true", out) -} - -func TestToTypedStringFromInt(t *testing.T) { - var out string - err := ToTyped(&out, dyn.V(123)) - require.NoError(t, err) - assert.Equal(t, "123", out) -} - -func TestToTypedStringFromFloat(t *testing.T) { - var out string - err := ToTyped(&out, dyn.V(1.2)) - require.NoError(t, err) - assert.Equal(t, "1.2", out) -} - -func TestToTypedBool(t *testing.T) { - var out bool - err := ToTyped(&out, dyn.V(true)) - require.NoError(t, err) - assert.True(t, out) -} - -func TestToTypedBoolOverwrite(t *testing.T) { - out := true - err := ToTyped(&out, dyn.V(false)) - require.NoError(t, err) - assert.False(t, out) -} - -func TestToTypedBoolFromString(t *testing.T) { - var out bool - - // True-ish - for _, v := range []string{"y", "yes", "on", "true", "True", "TRUE"} { - err := ToTyped(&out, dyn.V(v)) - require.NoError(t, err) - assert.True(t, out) - } - - // False-ish - for _, v := range []string{"n", "no", "off", "false", "False", "FALSE"} { - err := ToTyped(&out, dyn.V(v)) - require.NoError(t, err) - assert.False(t, out) - } - - // Other - err := ToTyped(&out, dyn.V("some other string")) - require.Error(t, err) -} - -func TestToTypedBoolFromStringVariableReference(t *testing.T) { - out := true - err := ToTyped(&out, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.False(t, out) -} - -func TestToTypedInt(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V(1234)) - require.NoError(t, err) - assert.Equal(t, int(1234), out) -} - -func TestToTypedInt32(t *testing.T) { - var out32 int32 - err := ToTyped(&out32, dyn.V(1235)) - require.NoError(t, err) - assert.Equal(t, int32(1235), out32) -} - -func TestToTypedInt64(t *testing.T) { - var out64 int64 - err := ToTyped(&out64, dyn.V(1236)) - require.NoError(t, err) - assert.Equal(t, int64(1236), out64) -} - -func TestToTypedIntOverwrite(t *testing.T) { - out := 123 - err := ToTyped(&out, dyn.V(1234)) - require.NoError(t, err) - assert.Equal(t, int(1234), out) -} - -func TestToTypedInt32Overwrite(t *testing.T) { - var out32 int32 = 123 - err := ToTyped(&out32, dyn.V(1234)) - require.NoError(t, err) - assert.Equal(t, int32(1234), out32) -} - -func TestToTypedInt64Overwrite(t *testing.T) { - var out64 int64 = 123 - err := ToTyped(&out64, dyn.V(1234)) - require.NoError(t, err) - assert.Equal(t, int64(1234), out64) -} - -func TestToTypedIntFromStringError(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V("abc")) - require.Error(t, err) -} - -func TestToTypedIntFromStringInt(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V("123")) - require.NoError(t, err) - assert.Equal(t, int(123), out) -} - -func TestToTypedIntFromStringVariableReference(t *testing.T) { - out := 123 - err := ToTyped(&out, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.Equal(t, int(0), out) -} - -func TestToTypedIntFromFloat(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V(1.0)) - require.NoError(t, err) - assert.Equal(t, int(1), out) -} - -func TestToTypedIntFromFloatError(t *testing.T) { - var out int - err := ToTyped(&out, dyn.V(1.2)) - require.ErrorContains(t, err, "expected an int, found a float") -} - -func TestToTypedFloat32(t *testing.T) { - var out float32 - err := ToTyped(&out, dyn.V(float32(1.0))) - require.NoError(t, err) - assert.Zero(t, 1.0-out) -} - -func TestToTypedFloat64(t *testing.T) { - var out float64 - err := ToTyped(&out, dyn.V(float64(1.0))) - require.NoError(t, err) - assert.Zero(t, 1.0-out) -} - -func TestToTypedFloat32Overwrite(t *testing.T) { - var out float32 = 1.0 - err := ToTyped(&out, dyn.V(float32(2.0))) - require.NoError(t, err) - assert.Zero(t, 2.0-out) -} - -func TestToTypedFloat64Overwrite(t *testing.T) { - out := 1.0 - err := ToTyped(&out, dyn.V(float64(2.0))) - require.NoError(t, err) - assert.Zero(t, 2.0-out) -} - -func TestToTypedFloat32FromStringError(t *testing.T) { - var out float32 - err := ToTyped(&out, dyn.V("abc")) - require.Error(t, err) -} - -func TestToTypedFloat64FromStringError(t *testing.T) { - var out float64 - err := ToTyped(&out, dyn.V("abc")) - require.Error(t, err) -} - -func TestToTypedFloat32FromString(t *testing.T) { - var out float32 - err := ToTyped(&out, dyn.V("1.2")) - require.NoError(t, err) - assert.Zero(t, 1.2-out) -} - -func TestToTypedFloat64FromString(t *testing.T) { - var out float64 - err := ToTyped(&out, dyn.V("1.2")) - require.NoError(t, err) - assert.Zero(t, 1.2-out) -} - -func TestToTypedFloat32FromStringVariableReference(t *testing.T) { - var out float32 = 1.0 - err := ToTyped(&out, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.Zero(t, out) -} - -func TestToTypedFloat64FromStringVariableReference(t *testing.T) { - out := 1.0 - err := ToTyped(&out, dyn.V("${var.foo}")) - require.NoError(t, err) - assert.Zero(t, out) -} - -func TestToTypedWithAliasKeyType(t *testing.T) { - type custom string - - var out map[custom]string - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Len(t, out, 2) - assert.Equal(t, "bar", out["foo"]) - assert.Equal(t, "baz", out["bar"]) -} - -func TestToTypedAnyWithBool(t *testing.T) { - var out any - err := ToTyped(&out, dyn.V(false)) - require.NoError(t, err) - assert.Equal(t, false, out) - - err = ToTyped(&out, dyn.V(true)) - require.NoError(t, err) - assert.Equal(t, true, out) -} - -func TestToTypedAnyWithMap(t *testing.T) { - var out any - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) - err := ToTyped(&out, v) - require.NoError(t, err) - assert.Equal(t, map[string]any{"foo": "bar", "bar": "baz"}, out) -} - -func TestToTypedAnyWithNil(t *testing.T) { - var out any - err := ToTyped(&out, dyn.NilValue) - require.NoError(t, err) - assert.Nil(t, out) -} - -func TestToTypedEmbeddedStructForceSendFields(t *testing.T) { - type Inner struct { - InnerField string `json:"inner_field"` - ForceSendFields []string `json:"-"` - } - - type Outer struct { - OuterField string `json:"outer_field"` - Inner - } - - var out Outer - m := dyn.Mapping{} - m.SetLoc("outer_field", nil, dyn.V("")) - m.SetLoc("inner_field", nil, dyn.V("")) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - // Bug: ForceSendFields contains "OuterField" which belongs to outer struct - // Expected: ForceSendFields should only contain "InnerField" - assert.Equal(t, []string{"InnerField"}, out.ForceSendFields) -} - -func TestToTypedMultipleEmbeddedStructsForceSendFields(t *testing.T) { - type First struct { - FirstField string `json:"first_field"` - ForceSendFields []string `json:"-"` - } - - type Second struct { - SecondField string `json:"second_field"` - ForceSendFields []string `json:"-"` - } - - type Outer struct { - OuterField string `json:"outer_field"` - First - Second - } - - var out Outer - m := dyn.Mapping{} - m.SetLoc("outer_field", nil, dyn.V("")) - m.SetLoc("first_field", nil, dyn.V("")) - m.SetLoc("second_field", nil, dyn.V("")) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - // Each embedded struct should only get its own fields in ForceSendFields - assert.Equal(t, []string{"FirstField"}, out.First.ForceSendFields) - assert.Equal(t, []string{"SecondField"}, out.Second.ForceSendFields) -} - -func TestToTypedMixedForceSendFields(t *testing.T) { - type First struct { - FirstField string `json:"first_field"` - // No ForceSendFields - } - - type Second struct { - SecondField string `json:"second_field"` - ForceSendFields []string `json:"-"` - } - - type Outer struct { - OuterField string `json:"outer_field"` - ForceSendFields []string `json:"-"` - First - Second - } - - var out Outer - m := dyn.Mapping{} - m.SetLoc("outer_field", nil, dyn.V("")) - m.SetLoc("first_field", nil, dyn.V("")) - m.SetLoc("second_field", nil, dyn.V("")) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - // Outer should get its own direct fields - assert.Equal(t, []string{"OuterField"}, out.ForceSendFields) - // First has no ForceSendFields field, so nothing to check - // Second should get its own field - assert.Equal(t, []string{"SecondField"}, out.Second.ForceSendFields) -} - -func TestToTypedFieldByNameBugRegressionTest(t *testing.T) { - // This test reproduces the EXACT bug: only direct field is zero, embedded fields are non-zero - // The bug occurred because dst.FieldByName("ForceSendFields") finds embedded ForceSendFields - // instead of direct ForceSendFields (which doesn't exist) - type BaseResource struct { - ID string `json:"id,omitempty"` - } - - type JobSettings struct { - Name string `json:"name"` - ForceSendFields []string `json:"-"` - } - - type Job struct { - BaseResource // embedded at [0] - JobSettings // embedded at [1], has ForceSendFields - Permissions []string `json:"permissions,omitempty"` // direct at [2] - } - - var out Job - m := dyn.Mapping{} - // Non-zero JobSettings fields - these should NOT go to ForceSendFields - m.SetLoc("name", nil, dyn.V("test-job")) - // Zero direct field - this SHOULD trigger the bug with old code - m.SetLoc("permissions", nil, dyn.V([]dyn.Value{})) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - // "Permissions" should be dropped, JobSettings.ForceSendFields stays empty - assert.Empty(t, out.ForceSendFields) - assert.Equal(t, "test-job", out.Name) - assert.Empty(t, out.Permissions) -} - -func TestToTypedDeeplyEmbeddedStructForceSendFields(t *testing.T) { - // Mirrors resources.PostgresProject -> PostgresProjectConfig -> ProjectSpec: - // Spec is embedded two levels down and Wrapper shadows ForceSendFields to keep - // its own direct field out of Spec's ForceSendFields. A zero-value spec field - // must route to Spec.ForceSendFields, not Wrapper's, otherwise the SDK marshaler - // fails with "field ... cannot be found in struct". - type Spec struct { - SpecField bool `json:"spec_field,omitempty"` - ForceSendFields []string `json:"-"` - } - - type Wrapper struct { - Spec - WrapperField string `json:"wrapper_field,omitempty"` - ForceSendFields []string `json:"-"` - } - - type Outer struct { - Wrapper - } - - var out Outer - m := dyn.Mapping{} - m.SetLoc("spec_field", nil, dyn.V(false)) - m.SetLoc("wrapper_field", nil, dyn.V("")) - v := dyn.V(m) - - err := ToTyped(&out, v) - require.NoError(t, err) - - assert.Equal(t, []string{"SpecField"}, out.Spec.ForceSendFields) - assert.Equal(t, []string{"WrapperField"}, out.ForceSendFields) -} diff --git a/libs/dyn/drop_keys.go b/libs/dyn/drop_keys.go deleted file mode 100644 index 494f9b9cd88..00000000000 --- a/libs/dyn/drop_keys.go +++ /dev/null @@ -1,27 +0,0 @@ -package dyn - -func DropKeys(v Value, drop []string) (Value, error) { - var err error - nv, err := Walk(v, func(p Path, v Value) (Value, error) { - if len(p) == 0 { - return v, nil - } - - // Check if this key should be dropped. - for _, key := range drop { - if p[0].Key() != key { - continue - } - - return InvalidValue, ErrDrop - } - - // Pass through all other values. - return v, ErrSkip - }) - if err != nil { - return InvalidValue, err - } - - return nv, nil -} diff --git a/libs/dyn/drop_keys_test.go b/libs/dyn/drop_keys_test.go deleted file mode 100644 index 83a9744ca65..00000000000 --- a/libs/dyn/drop_keys_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package dyn - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestDropKeysTest(t *testing.T) { - v := V(map[string]Value{ - "key1": V("value1"), - "key2": V("value2"), - "key3": V("value3"), - }) - - vout, err := DropKeys(v, []string{"key1", "key3"}) - require.NoError(t, err) - - mv := vout.MustMap() - require.Equal(t, 1, mv.Len()) - v, ok := mv.GetByString("key2") - require.True(t, ok) - require.Equal(t, "value2", v.MustString()) -} diff --git a/libs/dyn/dynassert/assert.go b/libs/dyn/dynassert/assert.go deleted file mode 100644 index 9e02eeff39c..00000000000 --- a/libs/dyn/dynassert/assert.go +++ /dev/null @@ -1,36 +0,0 @@ -package dynassert - -import ( - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -// Equal is a variant of assert.Equal that first converts dyn.Value to any. The practical consequence is that maps are compared -// without order. Another difference here is that locations are compared via Location() function, which drops all -// but first and get rids of nils. -func Equal(t assert.TestingT, expected, actual any, msgAndArgs ...any) bool { - ev, eok := expected.(dyn.Value) - av, aok := actual.(dyn.Value) - if eok && aok && ev.IsValid() && av.IsValid() { - if !assert.Equal(t, ev.AsAny(), av.AsAny(), msgAndArgs...) { - return false - } - - // The values are equal on contents. Now compare the locations. - if !assert.Equal(t, ev.Location(), av.Location(), msgAndArgs...) { - return false - } - - // Walk ev and av and compare the locations of each element. - _, err := dyn.Walk(ev, func(p dyn.Path, evv dyn.Value) (dyn.Value, error) { - avv, err := dyn.GetByPath(av, p) - if assert.NoError(t, err, "unable to get value from actual value at path %v", p.String()) { - assert.Equal(t, evv.Location(), avv.Location()) - } - return evv, nil - }) - return assert.NoError(t, err) - } - - return assert.Equal(t, expected, actual, msgAndArgs...) -} diff --git a/libs/dyn/dynvar/lookup.go b/libs/dyn/dynvar/lookup.go deleted file mode 100644 index 2bc08f47d47..00000000000 --- a/libs/dyn/dynvar/lookup.go +++ /dev/null @@ -1,21 +0,0 @@ -package dynvar - -import ( - "errors" - - "github.com/databricks/cli/libs/dyn" -) - -// Lookup is the type of lookup functions that can be used with [Resolve]. -type Lookup func(path dyn.Path) (dyn.Value, error) - -// ErrSkipResolution is returned by a lookup function to indicate that the -// resolution of a variable reference should be skipped. -var ErrSkipResolution = errors.New("skip resolution") - -// DefaultLookup is the default lookup function used by [Resolve]. -func DefaultLookup(in dyn.Value) Lookup { - return func(path dyn.Path) (dyn.Value, error) { - return dyn.GetByPath(in, path) - } -} diff --git a/libs/dyn/dynvar/lookup_test.go b/libs/dyn/dynvar/lookup_test.go deleted file mode 100644 index 2341d72084b..00000000000 --- a/libs/dyn/dynvar/lookup_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package dynvar_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/stretchr/testify/assert" -) - -func TestDefaultLookup(t *testing.T) { - lookup := dynvar.DefaultLookup(dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - })) - - v1, err := lookup(dyn.NewPath(dyn.Key("a"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V("a"), v1) - - v2, err := lookup(dyn.NewPath(dyn.Key("b"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V("b"), v2) - - _, err = lookup(dyn.NewPath(dyn.Key("c"))) - assert.True(t, dyn.IsNoSuchKeyError(err)) -} diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go deleted file mode 100644 index de25f44a212..00000000000 --- a/libs/dyn/dynvar/ref.go +++ /dev/null @@ -1,183 +0,0 @@ -package dynvar - -import ( - "fmt" - "regexp" - "strings" - - "github.com/databricks/cli/libs/dyn" -) - -// BaseVarDef matches a single dot-separated path segment in ${...} references. -// A segment may have leading underscores before the first letter, and may contain -// letters, digits, hyphens and underscores after that. -// Keep in sync with _base_var_def in python/databricks/bundles/core/_transform.py. -// Behavioral parity is enforced by testdata/reference_vectors.json. -const BaseVarDef = `_*\p{L}+([-_]*[\p{L}\p{N}]+)*` - -var re = regexp.MustCompile(fmt.Sprintf(`\$\{(%s(\.%s(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, BaseVarDef, BaseVarDef)) - -// Ref represents a variable reference. -// It is a string [dyn.Value] contained in a larger [dyn.Value]. -// Its path within the containing [dyn.Value] is also stored. -type Ref struct { - // Original value. - Value dyn.Value - - // String value in the original [dyn.Value]. - Str string - - // Matches of the variable reference in the string. - Matches [][]string - - // Byte offsets [start, end) of each entry in Matches within Str. Interpolation - // must substitute by offset: an escaped occurrence of the same reference text - // may appear earlier in Str, and a search-based replacement would hit that one. - Spans [][2]int -} - -// NewRef returns a new Ref if the given [dyn.Value] contains a string -// with one or more variable references. It returns false if the given -// [dyn.Value] does not contain variable references. -// -// A reference preceded by "$" is treated as an escaped literal and is not -// included in the returned Ref. For example, "$${foo}" contains no references -// because the "${foo}" part is escaped by the leading "$". -// -// Examples of a valid variable references: -// - "${a.b}" -// - "${a.b.c}" -// - "${a.b[0].c}" -// - "${a} ${b} ${c}" -func NewRef(v dyn.Value) (Ref, bool) { - s, ok := v.AsString() - if !ok { - return Ref{}, false - } - - // Find all matches with their byte positions so we can detect escaped references. - indices := re.FindAllStringSubmatchIndex(s, -1) - if len(indices) == 0 { - return Ref{}, false - } - - // Convert position-based matches to string matches, filtering out references - // escaped with a leading "$" (e.g. "$${foo}" should not match "${foo}"). - var m [][]string - var spans [][2]int - for _, idx := range indices { - if isEscapedAt(s, idx[0]) { - continue - } - match := make([]string, len(idx)/2) - for i := range len(idx) / 2 { - if idx[2*i] >= 0 { - match[i] = s[idx[2*i]:idx[2*i+1]] - } - } - m = append(m, match) - spans = append(spans, [2]int{idx[0], idx[1]}) - } - - if len(m) == 0 { - return Ref{}, false - } - - return Ref{ - Value: v, - Str: s, - Matches: m, - Spans: spans, - }, true -} - -// isEscapedAt reports whether the reference starting at byte offset start in s is -// escaped by an immediately preceding "$". -func isEscapedAt(s string, start int) bool { - return start > 0 && s[start-1] == '$' -} - -// Unescape turns escaped references into literal ones ("$${x}" -> "${x}"). Call it -// when building a value to send to the API: the extra "$" exists only to hide the -// reference from bundle variable resolution. -func Unescape(s string) string { - return strings.ReplaceAll(s, "$${", "${") -} - -// ReplaceRef substitutes every unescaped occurrence of reference (a full "${...}" -// string) in s with value. Escaped occurrences are left as they are, so -// strings.ReplaceAll is not a valid substitute: it would also rewrite the literal. -func ReplaceRef(s, reference, value string) string { - var sb strings.Builder - for pos := 0; ; { - i := strings.Index(s[pos:], reference) - if i < 0 { - sb.WriteString(s[pos:]) - return sb.String() - } - start := pos + i - sb.WriteString(s[pos:start]) - if isEscapedAt(s, start) { - sb.WriteString(reference) - } else { - sb.WriteString(value) - } - pos = start + len(reference) - } -} - -// IsPure returns true if the variable reference contains a single -// variable reference and nothing more. We need this so we can -// interpolate values of non-string types (i.e. it can be substituted). -func (v Ref) IsPure() bool { - // Need single match, equal to the incoming string. - if len(v.Matches) == 0 || len(v.Matches[0]) == 0 { - panic("invalid variable reference; expect at least one match") - } - return v.Matches[0][0] == v.Str -} - -func (v Ref) References() []string { - var out []string - for _, m := range v.Matches { - out = append(out, m[1]) - } - return out -} - -func IsPureVariableReference(s string) bool { - return len(s) > 0 && re.FindString(s) == s -} - -// ContainsVariableReference reports whether s contains at least one reference that -// is not escaped. Callers use it to decide whether a string still needs resolution, -// so an escaped "$${foo}" must not count: it is a literal, and treating it as -// pending leaves the value permanently unresolved. -func ContainsVariableReference(s string) bool { - for _, loc := range re.FindAllStringIndex(s, -1) { - if !isEscapedAt(s, loc[0]) { - return true - } - } - return false -} - -// If s is a pure variable reference, this function returns the corresponding -// dyn.Path. Otherwise, it returns false. -func PureReferenceToPath(s string) (dyn.Path, bool) { - ref, ok := NewRef(dyn.V(s)) - if !ok { - return nil, false - } - - if !ref.IsPure() { - return nil, false - } - - p, err := dyn.NewPathFromString(ref.References()[0]) - if err != nil { - return nil, false - } - - return p, true -} diff --git a/libs/dyn/dynvar/ref_test.go b/libs/dyn/dynvar/ref_test.go deleted file mode 100644 index 7b93ac633b9..00000000000 --- a/libs/dyn/dynvar/ref_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package dynvar - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNewRefNoString(t *testing.T) { - _, ok := NewRef(dyn.V(1)) - require.False(t, ok, "should not match non-string") -} - -func TestNewRefSpans(t *testing.T) { - // Spans must point at the unescaped occurrence, not the escaped one that - // appears earlier with identical text. - ref, ok := NewRef(dyn.V("$${a.b} ${a.b}")) - require.True(t, ok) - require.Len(t, ref.Matches, 1) - require.Len(t, ref.Spans, 1) - assert.Equal(t, "${a.b}", ref.Str[ref.Spans[0][0]:ref.Spans[0][1]]) - assert.Equal(t, 8, ref.Spans[0][0]) -} - -func TestContainsVariableReference(t *testing.T) { - tests := map[string]bool{ - "${a.b}": true, - "x ${a.b} y": true, - "$${a.b}": false, - "$${a.b} $${c.d}": false, - "$${a.b} ${c.d}": true, - "no refs": false, - "": false, - } - for in, want := range tests { - assert.Equal(t, want, ContainsVariableReference(in), "input %q", in) - } -} - -func TestUnescape(t *testing.T) { - assert.Equal(t, "${a.b}", Unescape("$${a.b}")) - assert.Equal(t, "${a} ${b}", Unescape("$${a} $${b}")) - assert.Equal(t, "${a.b}", Unescape("${a.b}"), "already unescaped is unchanged") - assert.Equal(t, "no refs", Unescape("no refs")) -} - -func TestReplaceRef(t *testing.T) { - tests := []struct { - in string - want string - }{ - {"${a.b}", "V"}, - {"$${a.b}", "$${a.b}"}, - {"$${a.b} ${a.b}", "$${a.b} V"}, - {"${a.b} $${a.b}", "V $${a.b}"}, - {"${a.b}-${a.b}", "V-V"}, - {"$${a.b} ${a.b} $${a.b}", "$${a.b} V $${a.b}"}, - {"untouched ${c.d}", "untouched ${c.d}"}, - } - for _, tt := range tests { - assert.Equal(t, tt.want, ReplaceRef(tt.in, "${a.b}", "V"), "input %q", tt.in) - } -} diff --git a/libs/dyn/dynvar/ref_vectors_test.go b/libs/dyn/dynvar/ref_vectors_test.go deleted file mode 100644 index d0084a6ae5c..00000000000 --- a/libs/dyn/dynvar/ref_vectors_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package dynvar - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type referenceVectorsFile struct { - Vectors []referenceVector `json:"vectors"` -} - -type referenceVector struct { - ID string `json:"id"` - Input string `json:"input"` - Match bool `json:"match"` - References []string `json:"references,omitempty"` - Pure *bool `json:"pure,omitempty"` - Path *string `json:"path,omitempty"` - PathOK *bool `json:"path_ok,omitempty"` -} - -func loadReferenceVectors(t *testing.T) []referenceVector { - t.Helper() - - b, err := os.ReadFile(filepath.Join("testdata", "reference_vectors.json")) - require.NoError(t, err) - - var file referenceVectorsFile - require.NoError(t, json.Unmarshal(b, &file)) - require.NotEmpty(t, file.Vectors) - - return file.Vectors -} - -func TestReferenceVectors(t *testing.T) { - for _, v := range loadReferenceVectors(t) { - t.Run(v.ID, func(t *testing.T) { - ref, ok := NewRef(dyn.V(v.Input)) - assert.Equal(t, v.Match, ok, "NewRef match") - - if v.Match { - require.NotEmpty(t, ref.References()) - if v.References != nil { - assert.Equal(t, v.References, ref.References(), "References") - } - } - - if v.Pure != nil { - assert.Equal(t, *v.Pure, IsPureVariableReference(v.Input), "IsPureVariableReference") - } - - switch { - case v.Path != nil: - path, ok := PureReferenceToPath(v.Input) - require.True(t, ok, "PureReferenceToPath") - assert.Equal(t, dyn.MustPathFromString(*v.Path), path) - case v.PathOK != nil: - _, ok := PureReferenceToPath(v.Input) - assert.Equal(t, *v.PathOK, ok, "PureReferenceToPath") - } - }) - } -} diff --git a/libs/dyn/dynvar/resolve.go b/libs/dyn/dynvar/resolve.go deleted file mode 100644 index 99981a2e51b..00000000000 --- a/libs/dyn/dynvar/resolve.go +++ /dev/null @@ -1,255 +0,0 @@ -package dynvar - -import ( - "errors" - "fmt" - "maps" - "slices" - "strings" - - "github.com/databricks/cli/libs/dyn" -) - -// Resolve resolves variable references in the given input value using the provided lookup function. -// It returns the resolved output value and any error encountered during the resolution process. -// -// For example, given the input value: -// -// { -// "a": "a", -// "b": "${a}", -// "c": "${b}${b}", -// } -// -// The output value will be: -// -// { -// "a": "a", -// "b": "a", -// "c": "aa", -// } -// -// If the input value contains a variable reference that cannot be resolved, an error is returned. -// If a cycle is detected in the variable references, an error is returned. -// If for some path the resolution function returns [ErrSkipResolution], the variable reference is left in place. -// This is useful when some variable references are not yet ready to be interpolated. -func Resolve(in dyn.Value, fn Lookup) (out dyn.Value, err error) { - return resolver{in: in, fn: fn}.run() -} - -// ReferenceError is returned for an unresolved variable reference. Suggestions -// are carried as data so callers (which can import libs/diag) format them. -type ReferenceError struct { - Reference string // original reference text, e.g. "var.hst" - Suggestions []string // corrected references, e.g. ["var.host", "var.hosts"] -} - -func (e *ReferenceError) Error() string { - return fmt.Sprintf("reference does not exist: ${%s}", e.Reference) -} - -type lookupResult struct { - v dyn.Value - err error -} - -type resolver struct { //nolint:recvcheck // value receiver for run(), pointer for mutation methods - in dyn.Value - fn Lookup - - refs map[string]Ref - resolved map[string]dyn.Value - - // Memoization for lookups. - lookups map[string]lookupResult -} - -func (r resolver) run() (out dyn.Value, err error) { - err = r.collectVariableReferences() - if err != nil { - return dyn.InvalidValue, err - } - - err = r.resolveVariableReferences() - if err != nil { - return dyn.InvalidValue, err - } - - out, err = r.replaceVariableReferences() - if err != nil { - return dyn.InvalidValue, err - } - - return out, nil -} - -func (r *resolver) collectVariableReferences() (err error) { - r.refs = make(map[string]Ref) - - // First walk the input to gather all values with a variable reference. - _, err = dyn.Walk(r.in, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - ref, ok := NewRef(v) - if !ok { - // Skip values without variable references. - return v, nil - } - - r.refs[p.String()] = ref - return v, nil - }) - - return err -} - -func (r *resolver) resolveVariableReferences() (err error) { - // Initialize cache for lookups. - r.lookups = make(map[string]lookupResult) - - // Initialize cache for resolved variable references. - r.resolved = make(map[string]dyn.Value) - - // Resolve each variable reference (in order). - // We sort the keys here to ensure that we always resolve the same variable reference first. - // This is done such that the cycle detection error is deterministic. If we did not do this, - // we could enter the cycle at any point in the cycle and return varying errors. - keys := slices.Sorted(maps.Keys(r.refs)) - for _, key := range keys { - v, err := r.resolveRef(r.refs[key], []string{key}) - if err != nil { - return err - } - r.resolved[key] = v - } - - return nil -} - -func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) { - // This is an unresolved variable reference. - deps := ref.References() - - // Resolve each of the dependencies, then interpolate them in the ref. - resolved := make([]dyn.Value, len(deps)) - complete := true - - for j, dep := range deps { - // Cycle detection. - if slices.Contains(seen, dep) { - return dyn.InvalidValue, fmt.Errorf( - "cycle detected in field resolution: %s", - strings.Join(append(seen, dep), " -> "), - ) - } - - v, err := r.resolveKey(dep, append(seen, dep)) - - // If we should skip resolution of this key, index j will hold an invalid [dyn.Value]. - if errors.Is(err, ErrSkipResolution) { - complete = false - continue - } else if err != nil { - // Otherwise, propagate the error. - return dyn.InvalidValue, err - } - - resolved[j] = v - } - - // Interpolate the resolved values. - if ref.IsPure() && complete { - // If the variable reference is pure, we can substitute it. - // This is useful for interpolating values of non-string types. - // - // Note: we use the location of the variable reference to preserve the information - // of where it is used. This also means that relative path resolution is done - // relative to where a variable is used, not where it is defined. - // - return dyn.NewValue(resolved[0].Value(), ref.Value.Locations()), nil - } - - // Not pure; perform string interpolation. - // - // Substitute by byte offset rather than by searching for the match text: the same - // reference may also appear escaped ("$${foo} ${foo}"), and a search would replace - // that occurrence instead, corrupting the literal and leaving the real one unresolved. - var sb strings.Builder - consumed := 0 - for j := range ref.Matches { - // The value is invalid if resolution returned [ErrSkipResolution]. - // We must skip those and leave the original variable reference in place, - // which happens naturally by not advancing past its span. - if !resolved[j].IsValid() { - continue - } - - // Try to turn the resolved value into a string. - s, ok := resolved[j].AsString() - if !ok { - // Only allow primitive types to be converted to string. - switch resolved[j].Kind() { - case dyn.KindString, dyn.KindBool, dyn.KindInt, dyn.KindFloat, dyn.KindTime, dyn.KindNil: - s = fmt.Sprint(resolved[j].AsAny()) - default: - return dyn.InvalidValue, fmt.Errorf("cannot interpolate non-primitive value of type %s into string", resolved[j].Kind()) - } - } - - start, end := ref.Spans[j][0], ref.Spans[j][1] - sb.WriteString(ref.Str[consumed:start]) - sb.WriteString(s) - consumed = end - } - sb.WriteString(ref.Str[consumed:]) - - return dyn.NewValue(sb.String(), ref.Value.Locations()), nil -} - -func (r *resolver) resolveKey(key string, seen []string) (dyn.Value, error) { - // Check if we have already looked up this key. - if v, ok := r.lookups[key]; ok { - return v.v, v.err - } - - // Parse the key into a path. - p, err := dyn.NewPathFromString(key) - if err != nil { - return dyn.InvalidValue, err - } - - // Look up the value for the given key. - v, err := r.fn(p) - if err != nil { - if dyn.IsNoSuchKeyError(err) { - // Carry suggestions as data; the caller formats them via libs/diag. - err = &ReferenceError{Reference: key, Suggestions: dyn.SuggestedReferences(err, key)} - } - - // Cache the return value and return to the caller. - r.lookups[key] = lookupResult{v: dyn.InvalidValue, err: err} - return dyn.InvalidValue, err - } - - // If the returned value is a valid variable reference, resolve it. - ref, ok := NewRef(v) - if ok { - v, err = r.resolveRef(ref, seen) - } - - // Cache the return value and return to the caller. - r.lookups[key] = lookupResult{v: v, err: err} - return v, err -} - -func (r *resolver) replaceVariableReferences() (dyn.Value, error) { - // Walk the input and replace all variable references. - return dyn.Walk(r.in, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - nv, ok := r.resolved[p.String()] - if !ok { - // No variable reference; return the original value. - return v, nil - } - - // We have a variable reference; return the resolved value. - return nv, nil - }) -} diff --git a/libs/dyn/dynvar/resolve_test.go b/libs/dyn/dynvar/resolve_test.go deleted file mode 100644 index c4b28e88d3f..00000000000 --- a/libs/dyn/dynvar/resolve_test.go +++ /dev/null @@ -1,430 +0,0 @@ -package dynvar_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func getByPath(t *testing.T, v dyn.Value, path string) dyn.Value { - v, err := dyn.Get(v, path) - require.NoError(t, err) - return v -} - -func TestResolve(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${a}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) -} - -func TestResolveNotFound(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "b": dyn.V("${a}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.ErrorContains(t, err, `reference does not exist: ${a}`) -} - -func TestResolveNotFoundSuggestsCloseKey(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "host": dyn.V("example.com"), - "b": dyn.V("${hst}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.ErrorContains(t, err, "reference does not exist: ${hst}") - - var refErr *dynvar.ReferenceError - require.ErrorAs(t, err, &refErr) - assert.Equal(t, []string{"host"}, refErr.Suggestions) -} - -func TestResolveWithNesting(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("${f.a}"), - "f": dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${f.a}"), - }), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "f.a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "f.b").MustString()) -} - -func TestResolveWithRecursion(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${b}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) -} - -func TestResolveWithRecursionLoop(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${c}"), - "c": dyn.V("${d}"), - "d": dyn.V("${b}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - assert.ErrorContains(t, err, "cycle detected in field resolution: b -> c -> d -> b") -} - -func TestResolveWithRecursionLoopSelf(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("${a}"), - }) - - _, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - assert.ErrorContains(t, err, "cycle detected in field resolution: a -> a") -} - -func TestResolveWithStringConcatenation(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - "c": dyn.V("${a}${b}${a}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "b", getByPath(t, out, "b").MustString()) - assert.Equal(t, "aba", getByPath(t, out, "c").MustString()) -} - -func TestResolveWithEscapedReference(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("A"), - "escaped_only": dyn.V("$${a}"), - "escaped_first": dyn.V("$${a} ${a}"), - "real_first": dyn.V("${a} $${a}"), - "surrounded": dyn.V("$${a} ${a} $${a}"), - "other_name": dyn.V("$${zz} ${a}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // The escape is preserved here; it is removed when the API payload is built. - assert.Equal(t, "$${a}", getByPath(t, out, "escaped_only").MustString()) - assert.Equal(t, "$${a} A", getByPath(t, out, "escaped_first").MustString()) - assert.Equal(t, "A $${a}", getByPath(t, out, "real_first").MustString()) - assert.Equal(t, "$${a} A $${a}", getByPath(t, out, "surrounded").MustString()) - assert.Equal(t, "$${zz} A", getByPath(t, out, "other_name").MustString()) -} - -func TestResolveWithTypeInterpolation(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V(1), - "b": dyn.V(2), - "c": dyn.V("${a} ${b}"), - "float_val": dyn.V(3.14), - "bool_true": dyn.V(true), - "bool_false": dyn.V(false), - "time_val": dyn.V(dyn.MustTime("2024-01-01")), - "nil_val": dyn.NilValue, - // Test interpolation of different types in string templates - "float_interp": dyn.V("Value: ${float_val}"), - "bool_true_interp": dyn.V("Enabled: ${bool_true}"), - "bool_false_interp": dyn.V("Disabled: ${bool_false}"), - "time_interp": dyn.V("Date: ${time_val}"), - "nil_interp": dyn.V("Null: ${nil_val}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // Integer interpolation - assert.Equal(t, "1 2", getByPath(t, out, "c").MustString()) - - // Float interpolation - assert.Equal(t, "Value: 3.14", getByPath(t, out, "float_interp").MustString()) - - // Bool interpolation - assert.Equal(t, "Enabled: true", getByPath(t, out, "bool_true_interp").MustString()) - assert.Equal(t, "Disabled: false", getByPath(t, out, "bool_false_interp").MustString()) - - // Time interpolation should convert to string representation of time.Time - // Note: time.Time string representation includes timezone info - timeResult := getByPath(t, out, "time_interp").MustString() - assert.Contains(t, timeResult, "Date: 2024-01-01") - assert.Contains(t, timeResult, "00:00:00") - - // Nil interpolation - assert.Equal(t, "Null: ", getByPath(t, out, "nil_interp").MustString()) -} - -func TestResolveWithTypeRetentionFailure(t *testing.T) { - // Test that mapping interpolation fails with an error - mappingTest := dyn.V(map[string]dyn.Value{ - "mapping": dyn.V(map[string]dyn.Value{"key": dyn.V("value")}), - "interp": dyn.V("Config: ${mapping}"), - }) - _, err := dynvar.Resolve(mappingTest, dynvar.DefaultLookup(mappingTest)) - require.Error(t, err) - assert.Equal(t, "cannot interpolate non-primitive value of type map into string", err.Error()) -} - -func TestResolveWithTypeRetention(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "int": dyn.V(1), - "int_var": dyn.V("${int}"), - "bool_true": dyn.V(true), - "bool_true_var": dyn.V("${bool_true}"), - "bool_false": dyn.V(false), - "bool_false_var": dyn.V("${bool_false}"), - "float": dyn.V(1.0), - "float_var": dyn.V("${float}"), - "string": dyn.V("a"), - "string_var": dyn.V("${string}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.EqualValues(t, 1, getByPath(t, out, "int").MustInt()) - assert.EqualValues(t, 1, getByPath(t, out, "int_var").MustInt()) - - assert.True(t, getByPath(t, out, "bool_true").MustBool()) - assert.True(t, getByPath(t, out, "bool_true_var").MustBool()) - - assert.False(t, getByPath(t, out, "bool_false").MustBool()) - assert.False(t, getByPath(t, out, "bool_false_var").MustBool()) - - assert.Zero(t, 1.0-getByPath(t, out, "float").MustFloat()) - assert.Zero(t, 1.0-getByPath(t, out, "float_var").MustFloat()) - - assert.Equal(t, "a", getByPath(t, out, "string").MustString()) - assert.Equal(t, "a", getByPath(t, out, "string_var").MustString()) -} - -func TestResolveWithSkip(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - "c": dyn.V("${a}"), - "d": dyn.V("${b}"), - "e": dyn.V("${a} ${b}"), - "f": dyn.V("${b} ${a} ${a} ${b}"), - }) - - fallback := dynvar.DefaultLookup(in) - ignore := func(path dyn.Path) (dyn.Value, error) { - // If the variable reference to look up starts with "b", skip it. - if path.HasPrefix(dyn.NewPath(dyn.Key("b"))) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - } - return fallback(path) - } - - out, err := dynvar.Resolve(in, ignore) - require.NoError(t, err) - - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "b", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) - - // Check that the skipped variable references are not interpolated. - assert.Equal(t, "${b}", getByPath(t, out, "d").MustString()) - assert.Equal(t, "a ${b}", getByPath(t, out, "e").MustString()) - assert.Equal(t, "${b} a a ${b}", getByPath(t, out, "f").MustString()) //nolint:dupword -} - -func TestResolveWithSkipEverything(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("b"), - "c": dyn.V("${a}"), - "d": dyn.V("${b}"), - "e": dyn.V("${a} ${b}"), - "f": dyn.V("${b} ${a} ${a} ${b}"), - "g": dyn.V("${d} ${c} ${c} ${d}"), - }) - - // The call must not replace anything if the lookup function returns ErrSkipResolution. - out, err := dynvar.Resolve(in, func(path dyn.Path) (dyn.Value, error) { - return dyn.InvalidValue, dynvar.ErrSkipResolution - }) - require.NoError(t, err) - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "b", getByPath(t, out, "b").MustString()) - assert.Equal(t, "${a}", getByPath(t, out, "c").MustString()) - assert.Equal(t, "${b}", getByPath(t, out, "d").MustString()) - assert.Equal(t, "${a} ${b}", getByPath(t, out, "e").MustString()) - assert.Equal(t, "${b} ${a} ${a} ${b}", getByPath(t, out, "f").MustString()) - assert.Equal(t, "${d} ${c} ${c} ${d}", getByPath(t, out, "g").MustString()) -} - -func TestResolveWithInterpolateNewRef(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - }) - - // The call replaces ${a} with ${foobar} and skips everything else. - out, err := dynvar.Resolve(in, func(path dyn.Path) (dyn.Value, error) { - if path.String() == "a" { - return dyn.V("${foobar}"), nil - } - return dyn.InvalidValue, dynvar.ErrSkipResolution - }) - - require.NoError(t, err) - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "${foobar}", getByPath(t, out, "b").MustString()) -} - -func TestResolveWithInterpolateAliasedRef(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "a": dyn.V("a"), - "b": dyn.V("${a}"), - "c": dyn.V("${x}"), - }) - - // The call replaces ${x} with ${b} and skips everything else. - out, err := dynvar.Resolve(in, func(path dyn.Path) (dyn.Value, error) { - if path.String() == "x" { - return dyn.V("${b}"), nil - } - return dyn.GetByPath(in, path) - }) - - require.NoError(t, err) - assert.Equal(t, "a", getByPath(t, out, "a").MustString()) - assert.Equal(t, "a", getByPath(t, out, "b").MustString()) - assert.Equal(t, "a", getByPath(t, out, "c").MustString()) -} - -func TestResolveIndexedRefs(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{dyn.V("a"), dyn.V("b")}), - "a": dyn.V("a: ${slice[0]}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveIndexedRefsFromMap(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "map": dyn.V( - map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{dyn.V("a")}), - }), - "a": dyn.V("a: ${map.slice[0]}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveMapFieldFromIndexedRefs(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "map": dyn.V( - map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "value": dyn.V("a"), - }), - }), - }), - "a": dyn.V("a: ${map.slice[0].value}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveNestedIndexedRefs(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "slice": dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{dyn.V("a")}), - }), - "a": dyn.V("a: ${slice[0][0]}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - assert.Equal(t, "a: a", getByPath(t, out, "a").MustString()) -} - -func TestResolveMapVariable(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "map": dyn.V(map[string]dyn.Value{ - "key1": dyn.V("value1"), - "key2": dyn.V("value2"), - }), - "var": dyn.V("${map}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // Verify that the map variable was interpolated correctly - mapVal := getByPath(t, out, "var") - _, ok := mapVal.AsMap() - require.True(t, ok, "expected map value") - - // Verify the map contents - assert.Equal(t, "value1", getByPath(t, mapVal, "key1").MustString()) - assert.Equal(t, "value2", getByPath(t, mapVal, "key2").MustString()) -} - -func TestResolveSequenceVariable(t *testing.T) { - in := dyn.V(map[string]dyn.Value{ - "seq": dyn.V([]dyn.Value{ - dyn.V("value1"), - dyn.V("value2"), - }), - "var": dyn.V("${seq}"), - }) - - out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) - require.NoError(t, err) - - // Verify that the sequence variable was interpolated correctly - seqVal := getByPath(t, out, "var") - seq, ok := seqVal.AsSequence() - require.True(t, ok, "expected sequence value") - require.Len(t, seq, 2) - - // Verify the sequence contents - assert.Equal(t, "value1", seq[0].MustString()) - assert.Equal(t, "value2", seq[1].MustString()) -} diff --git a/libs/dyn/dynvar/testdata/reference_vectors.json b/libs/dyn/dynvar/testdata/reference_vectors.json deleted file mode 100644 index 2c47b135450..00000000000 --- a/libs/dyn/dynvar/testdata/reference_vectors.json +++ /dev/null @@ -1,368 +0,0 @@ -{ - "comment": "Shared contract for ${...} reference parsing. Consumed by libs/dyn/dynvar and python/databricks/bundles/core/_transform.py tests.", - "vectors": [ - { - "id": "ascii-simple", - "input": "${hello_world.world_world}", - "match": true, - "references": ["hello_world.world_world"], - "pure": true, - "path": "hello_world.world_world" - }, - { - "id": "ascii-hyphen-separators", - "input": "${helloworld.world-world}", - "match": true, - "references": ["helloworld.world-world"], - "pure": true, - "path": "helloworld.world-world" - }, - { - "id": "ascii-hyphen-in-segment", - "input": "${hello-world.world-world}", - "match": true, - "references": ["hello-world.world-world"], - "pure": true, - "path": "hello-world.world-world" - }, - { - "id": "ascii-underscore-separators", - "input": "${hello_world.world__world}", - "match": true, - "references": ["hello_world.world__world"], - "pure": true, - "path": "hello_world.world__world" - }, - { - "id": "ascii-double-hyphen", - "input": "${hello_world.world--world}", - "match": true, - "references": ["hello_world.world--world"], - "pure": true, - "path": "hello_world.world--world" - }, - { - "id": "ascii-mixed-separators-1", - "input": "${hello_world.world-_world}", - "match": true, - "references": ["hello_world.world-_world"], - "pure": true, - "path": "hello_world.world-_world" - }, - { - "id": "ascii-mixed-separators-2", - "input": "${hello_world.world_-world}", - "match": true, - "references": ["hello_world.world_-world"], - "pure": true, - "path": "hello_world.world_-world" - }, - { - "id": "unicode-accent-segment", - "input": "${café.bar}", - "match": true, - "references": ["café.bar"], - "pure": true, - "path": "café.bar" - }, - { - "id": "unicode-accent-trailing-segment", - "input": "${hello.café}", - "match": true, - "references": ["hello.café"], - "pure": true, - "path": "hello.café" - }, - { - "id": "unicode-cjk", - "input": "${变量.世界}", - "match": true, - "references": ["变量.世界"], - "pure": true, - "path": "变量.世界" - }, - { - "id": "unicode-cjk-numeral-letter", - "input": "${变量.九世界}", - "match": true, - "references": ["变量.九世界"], - "pure": true, - "path": "变量.九世界" - }, - { - "id": "unicode-arabic", - "input": "${العربية.key}", - "match": true, - "references": ["العربية.key"], - "pure": true, - "path": "العربية.key" - }, - { - "id": "unicode-multi-ref", - "input": "${变量.世界} ${café.bar}", - "match": true, - "references": ["变量.世界", "café.bar"], - "pure": false - }, - { - "id": "unicode-var-reference", - "input": "${var.变量}", - "match": true, - "references": ["var.变量"], - "pure": true, - "path": "var.变量" - }, - { - "id": "unicode-resource-reference", - "input": "${resources.jobs.工作.name}", - "match": true, - "references": ["resources.jobs.工作.name"], - "pure": true, - "path": "resources.jobs.工作.name" - }, - { - "id": "indexed-path", - "input": "${foo.bar.baz[0]}", - "match": true, - "references": ["foo.bar.baz[0]"], - "pure": true, - "path": "foo.bar.baz[0]" - }, - { - "id": "nested-indexed-path", - "input": "${foo.bar.baz[0][1].qux}", - "match": true, - "references": ["foo.bar.baz[0][1].qux"], - "pure": true, - "path": "foo.bar.baz[0][1].qux" - }, - { - "id": "indexed-unicode-path", - "input": "${变量.世界[0]}", - "match": true, - "references": ["变量.世界[0]"], - "pure": true, - "path": "变量.世界[0]" - }, - { - "id": "python-simple-variable", - "input": "${var.my_var}", - "match": true, - "references": ["var.my_var"], - "pure": true, - "path": "var.my_var" - }, - { - "id": "python-indexed-variable", - "input": "${var.my_var[0].foo}", - "match": true, - "references": ["var.my_var[0].foo"], - "pure": true, - "path": "var.my_var[0].foo" - }, - { - "id": "leading-underscore", - "input": "${_hello_world.world_world}", - "match": true, - "references": ["_hello_world.world_world"], - "pure": true, - "path": "_hello_world.world_world" - }, - { - "id": "resource-key-leading-underscore", - "input": "${resources.jobs._job.id}", - "match": true, - "references": ["resources.jobs._job.id"], - "pure": true, - "path": "resources.jobs._job.id" - }, - { - "id": "resource-key-leading-underscore-and-hyphens", - "input": "${resources.jobs._asset-bundle-deploymentjob.id}", - "match": true, - "references": ["resources.jobs._asset-bundle-deploymentjob.id"], - "pure": true, - "path": "resources.jobs._asset-bundle-deploymentjob.id" - }, - { - "id": "segment-trailing-hyphen", - "input": "${hello_world-.world_world}", - "match": false - }, - { - "id": "segment-trailing-underscore", - "input": "${hello_world-_.world_world}", - "match": false - }, - { - "id": "second-segment-trailing-hyphen", - "input": "${helloworld.world-world-}", - "match": false - }, - { - "id": "first-segment-trailing-hyphen", - "input": "${helloworld-.world-world}", - "match": false - }, - { - "id": "second-segment-leading-hyphen", - "input": "${helloworld.-world-world}", - "match": false - }, - { - "id": "leading-and-trailing-hyphen", - "input": "${-hello-world.-world-world-}", - "match": false - }, - { - "id": "invalid-separator-sequence", - "input": "${_-_._-_.id}", - "match": false - }, - { - "id": "leading-digit-ascii", - "input": "${0helloworld.world-world}", - "match": false - }, - { - "id": "leading-digit-second-segment", - "input": "${helloworld.9world-world}", - "match": false - }, - { - "id": "leading-digit-unicode", - "input": "${0café.bar}", - "match": false - }, - { - "id": "leading-digit-cjk", - "input": "${变量.9世界}", - "match": false - }, - { - "id": "leading-hyphen-cjk", - "input": "${变量.-世界}", - "match": false - }, - { - "id": "trailing-hyphen-accent", - "input": "${café-}", - "match": false - }, - { - "id": "nfd-decomposed-accent", - "comment": "NFD e + combining acute; not equivalent to NFC café", - "input": "${cafe\u0301.bar}", - "match": false - }, - { - "id": "emoji-in-segment", - "input": "${foo.🌍}", - "match": false - }, - { - "id": "emoji-leading-segment", - "input": "${🌍.bar}", - "match": false - }, - { - "id": "symbol-in-segment", - "input": "${foo.bar$}", - "match": false - }, - { - "id": "zero-width-space", - "input": "${foo.bar\u200b}", - "match": false - }, - { - "id": "fullwidth-digit", - "input": "${变量.９世界}", - "match": false - }, - { - "id": "empty-string", - "input": "", - "match": false, - "pure": false - }, - { - "id": "suffix-after-reference", - "input": "${foo.bar} suffix", - "match": true, - "references": ["foo.bar"], - "pure": false, - "path_ok": false - }, - { - "id": "prefix-before-reference", - "input": "prefix ${foo.bar}", - "match": true, - "references": ["foo.bar"], - "pure": false, - "path_ok": false - }, - { - "id": "adjacent-references", - "input": "${foo.one}${foo.two}", - "match": true, - "references": ["foo.one", "foo.two"], - "pure": false, - "path_ok": false - }, - { - "id": "multiple-variables-not-pure", - "input": "${var.my_var} ${var.my_var}", - "match": true, - "references": ["var.my_var", "var.my_var"], - "pure": false, - "path_ok": false - }, - { - "id": "unclosed-reference", - "input": "${foo.bar", - "match": false - }, - { - "id": "stray-closing-brace", - "input": "foo.bar}", - "match": false - }, - { - "id": "bare-path", - "input": "foo.bar", - "match": false - }, - { - "id": "wrong-braces", - "input": "{foo.bar}", - "match": false - }, - { - "id": "escaped-double-dollar", - "input": "$${foo.bar}", - "match": false - }, - { - "id": "escaped-only-multiple", - "input": "$${a.b} $${c.d}", - "match": false - }, - { - "id": "escaped-mixed-with-real", - "input": "${real.ref} $${escaped.ref}", - "match": true, - "references": ["real.ref"], - "pure": false, - "path_ok": false - }, - { - "id": "escaped-prefix-real-suffix", - "input": "$${escaped} text ${real.ref}", - "match": true, - "references": ["real.ref"], - "pure": false, - "path_ok": false - } - ] -} diff --git a/libs/dyn/jsonloader/json.go b/libs/dyn/jsonloader/json.go deleted file mode 100644 index ffb715e9c9c..00000000000 --- a/libs/dyn/jsonloader/json.go +++ /dev/null @@ -1,134 +0,0 @@ -package jsonloader - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" -) - -func LoadJSON(data []byte, source string) (dyn.Value, error) { - offset := BuildLineOffsets(data) - offset.SetSource(source) - - reader := bytes.NewReader(data) - decoder := json.NewDecoder(reader) - - // Use json.Number to avoid losing precision on int64 values above 2^53 (e.g. job and run IDs). - decoder.UseNumber() - - // Start decoding from the top-level value - value, err := decodeValue(decoder, &offset) - if err != nil { - if errors.Is(err, io.EOF) { - err = errors.New("unexpected end of JSON input") - } - // Get the current offset for error reporting - errorOffset := decoder.InputOffset() - errorLocation := offset.GetPosition(errorOffset) - return dyn.InvalidValue, fmt.Errorf("error decoding JSON at %s: %w", errorLocation, err) - } - - // Check if there are any remaining tokens (should not be valid JSON) - if decoder.More() { - errorOffset := decoder.InputOffset() - errorLocation := offset.GetPosition(errorOffset) - return dyn.InvalidValue, fmt.Errorf("error decoding JSON at %s: unexpected additional content", errorLocation) - } - - return value, nil -} - -func decodeValue(decoder *json.Decoder, o *Offset) (dyn.Value, error) { - // Read the next JSON token - token, err := decoder.Token() - if err != nil { - return dyn.InvalidValue, err - } - - // Get the current byte offset and the location. - // We will later use this location to store the location of the value in the file - // For objects and arrays, we will store the location of the opening '{' or '[' - // For primitive types, we will store the location of the value itself (end of the value) - // We can't reliably calculate the beginning of the value for primitive types because - // the decoder doesn't provide the offset of the beginning of the value and the value might or might not be quoted. - offset := decoder.InputOffset() - location := o.GetPosition(offset) - - switch tok := token.(type) { - case json.Delim: - switch tok { - case '{': - location = o.GetPosition(offset - 1) - // Decode JSON object - obj := dyn.NewMapping() - for decoder.More() { - // Decode the key - keyToken, err := decoder.Token() - if err != nil { - return invalidValueWithLocation(decoder, o), err - } - key, ok := keyToken.(string) - if !ok { - return invalidValueWithLocation(decoder, o), errors.New("expected string for object key") - } - - // Get the offset of the key by subtracting the length of the key and the '"' character - keyOffset := decoder.InputOffset() - int64(len(key)+1) - loc := []diag.Location{o.GetPosition(keyOffset)} - - // Decode the value recursively - val, err := decodeValue(decoder, o) - if err != nil { - return invalidValueWithLocation(decoder, o), err - } - - obj.SetLoc(key, loc, val) - } - // Consume the closing '}' - if _, err := decoder.Token(); err != nil { - return invalidValueWithLocation(decoder, o), err - } - return dyn.NewValue(obj, []diag.Location{location}), nil - case '[': - location = o.GetPosition(offset - 1) - // Decode JSON array - var arr []dyn.Value - for decoder.More() { - val, err := decodeValue(decoder, o) - if err != nil { - return invalidValueWithLocation(decoder, o), err - } - arr = append(arr, val) - } - // Consume the closing ']' - if _, err := decoder.Token(); err != nil { - return invalidValueWithLocation(decoder, o), err - } - return dyn.NewValue(arr, []diag.Location{location}), nil - } - case json.Number: - // Integers that overflow int64 fall back to float64, matching the decoder's behavior without UseNumber. - if i64, err := tok.Int64(); err == nil { - return dyn.NewValue(i64, []diag.Location{location}), nil - } - f64, err := tok.Float64() - if err != nil { - return invalidValueWithLocation(decoder, o), fmt.Errorf("invalid number %q: %w", tok.String(), err) - } - return dyn.NewValue(f64, []diag.Location{location}), nil - default: - return dyn.NewValue(tok, []diag.Location{location}), nil - } - - return invalidValueWithLocation(decoder, o), fmt.Errorf("unexpected token: %v", token) -} - -func invalidValueWithLocation(decoder *json.Decoder, o *Offset) dyn.Value { - location := o.GetPosition(decoder.InputOffset()) - return dyn.InvalidValue.WithLocations([]diag.Location{location}) -} diff --git a/libs/dyn/jsonloader/json_test.go b/libs/dyn/jsonloader/json_test.go deleted file mode 100644 index e274de559ce..00000000000 --- a/libs/dyn/jsonloader/json_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package jsonloader - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const jsonData = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx", - "email_notifications": { - "on_start": [], - "on_success": [], - "on_failure": [] - }, - "webhook_notifications": { - "on_start": [], - "on_failure": [] - }, - "notification_settings": { - "no_alert_for_skipped_runs": true, - "no_alert_for_canceled_runs": true - }, - "timeout_seconds": 0, - "max_concurrent_runs": 1, - "tasks": [ - { - "task_key": "xxx", - "email_notifications": {}, - "notification_settings": {}, - "timeout_seconds": 0, - "max_retries": 0, - "min_retry_interval_millis": 0, - "retry_on_timeout": "true" - } - ] - } -} -` - -func TestJsonLoader(t *testing.T) { - v, err := LoadJSON([]byte(jsonData), "(inline)") - assert.NoError(t, err) - - var r jobs.ResetJob - err = convert.ToTyped(&r, v) - assert.NoError(t, err) -} - -const malformedMap = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx", - "wrong", - } -} -` - -func TestJsonLoaderMalformedMap(t *testing.T) { - _, err := LoadJSON([]byte(malformedMap), "(inline)") - assert.ErrorContains(t, err, "error decoding JSON at (inline):6:16: invalid character ',' after object key") -} - -const malformedArray = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx", - "tasks": [1, "asd",] - } -}` - -func TestJsonLoaderMalformedArray(t *testing.T) { - _, err := LoadJSON([]byte(malformedArray), "path/to/file.json") - assert.ErrorContains(t, err, "error decoding JSON at path/to/file.json:6:27: invalid character ',' looking for beginning of value") -} - -const eofData = ` -{ - "job_id": 123, - "new_settings": { - "name": "xxx",` - -func TestJsonLoaderEOF(t *testing.T) { - _, err := LoadJSON([]byte(eofData), "path/to/file.json") - assert.ErrorContains(t, err, "unexpected end of JSON input") -} - -const mapWithNoBraces = ` -"job_id": 123, -"new_settings": { - "name": "xxx", - "wrong": "xxx", -} -` - -func TestJsonMapWithoutBraces(t *testing.T) { - _, err := LoadJSON([]byte(mapWithNoBraces), "path/to/file.json") - assert.ErrorContains(t, err, "error decoding JSON at") -} - -const validInline = `["job_id", 123]` - -func TestJsonValidInline(t *testing.T) { - _, err := LoadJSON([]byte(validInline), "path/to/file.json") - assert.NoError(t, err) -} - -func TestJsonLoaderNumbers(t *testing.T) { - for _, tc := range []struct { - input string - expected any - }{ - {`123`, int64(123)}, - {`-1`, int64(-1)}, - {`123456789012345678`, int64(123456789012345678)}, - {`-123456789012345678`, int64(-123456789012345678)}, - {`9223372036854775807`, int64(9223372036854775807)}, - {`2.0`, 2.0}, - {`2.5`, 2.5}, - {`1e3`, 1000.0}, - {`18446744073709551615`, 1.8446744073709552e+19}, - } { - v, err := LoadJSON([]byte(tc.input), "(inline)") - assert.NoError(t, err, tc.input) - assert.Equal(t, tc.expected, v.AsAny(), tc.input) - } -} - -func TestJsonLoaderNumberOutOfRange(t *testing.T) { - _, err := LoadJSON([]byte(`1e400`), "(inline)") - assert.ErrorContains(t, err, "value out of range") -} - -const mixedNumbersData = ` -{ - "job_id": 123456789012345678, - "new_settings": { - "name": "xxx", - "timeout_seconds": 100 - } -} -` - -func TestJsonLoaderMixedNumbersToTyped(t *testing.T) { - v, err := LoadJSON([]byte(mixedNumbersData), "(inline)") - require.NoError(t, err) - - var r jobs.ResetJob - err = convert.ToTyped(&r, v) - require.NoError(t, err) - assert.Equal(t, int64(123456789012345678), r.JobId) - assert.Equal(t, 100, r.NewSettings.TimeoutSeconds) -} diff --git a/libs/dyn/jsonloader/locations.go b/libs/dyn/jsonloader/locations.go deleted file mode 100644 index 3dbe2da6630..00000000000 --- a/libs/dyn/jsonloader/locations.go +++ /dev/null @@ -1,49 +0,0 @@ -package jsonloader - -import ( - "sort" - - "github.com/databricks/cli/libs/diag" -) - -type LineOffset struct { - Line int - Start int64 -} - -type Offset struct { //nolint:recvcheck // value receiver for read-only GetPosition, pointer for SetSource - offsets []LineOffset - source string -} - -// buildLineOffsets scans the input data and records the starting byte offset of each line. -func BuildLineOffsets(data []byte) Offset { - offsets := []LineOffset{{Line: 1, Start: 0}} - line := 1 - for i, b := range data { - if b == '\n' { - line++ - offsets = append(offsets, LineOffset{Line: line, Start: int64(i + 1)}) - } - } - return Offset{offsets: offsets} -} - -// GetPosition maps a byte offset to its corresponding line and column numbers. -func (o Offset) GetPosition(offset int64) diag.Location { - // Binary search to find the line - idx := max(sort.Search(len(o.offsets), func(i int) bool { - return o.offsets[i].Start > offset - })-1, 0) - - lineOffset := o.offsets[idx] - return diag.Location{ - File: o.source, - Line: lineOffset.Line, - Column: int(offset-lineOffset.Start) + 1, - } -} - -func (o *Offset) SetSource(source string) { - o.source = source -} diff --git a/libs/dyn/kind.go b/libs/dyn/kind.go deleted file mode 100644 index 1890e9e2ccd..00000000000 --- a/libs/dyn/kind.go +++ /dev/null @@ -1,70 +0,0 @@ -package dyn - -import ( - "fmt" -) - -type Kind int - -const ( - // Invalid is the zero value of Kind. - KindInvalid Kind = iota - KindMap - KindSequence - KindString - KindBool - KindInt - KindFloat - KindTime - KindNil -) - -func kindOf(v any) Kind { - switch v.(type) { - case Mapping: - return KindMap - case []Value: - return KindSequence - case string: - return KindString - case bool: - return KindBool - case int, int8, int16, int32, int64: - return KindInt - case uint, uint8, uint16, uint32, uint64: - return KindInt - case float32, float64: - return KindFloat - case Time: - return KindTime - case nil: - return KindNil - default: - panic(fmt.Sprintf("not handled: %T", v)) - } -} - -func (k Kind) String() string { - switch k { - case KindInvalid: - return "invalid" - case KindMap: - return "map" - case KindSequence: - return "sequence" - case KindString: - return "string" - case KindBool: - return "bool" - case KindInt: - return "int" - case KindFloat: - return "float" - case KindTime: - return "time" - case KindNil: - return "nil" - default: - panic(fmt.Sprintf("invalid kind value: %d", k)) - } -} diff --git a/libs/dyn/kind_test.go b/libs/dyn/kind_test.go deleted file mode 100644 index 84c90713fb8..00000000000 --- a/libs/dyn/kind_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestKindZeroValue(t *testing.T) { - // Assert that the zero value of [dyn.Kind] is the invalid kind. - var k dyn.Kind - assert.Equal(t, dyn.KindInvalid, k) -} - -func TestKindToString(t *testing.T) { - for _, tt := range []struct { - k dyn.Kind - s string - }{ - {dyn.KindInvalid, "invalid"}, - {dyn.KindMap, "map"}, - {dyn.KindSequence, "sequence"}, - {dyn.KindString, "string"}, - {dyn.KindBool, "bool"}, - {dyn.KindInt, "int"}, - {dyn.KindFloat, "float"}, - {dyn.KindTime, "time"}, - {dyn.KindNil, "nil"}, - } { - assert.Equal(t, tt.s, tt.k.String()) - } - - // Panic on unknown kind. - assert.PanicsWithValue(t, "invalid kind value: 100", func() { - _ = dyn.Kind(100).String() - }) -} diff --git a/libs/dyn/mapping.go b/libs/dyn/mapping.go deleted file mode 100644 index 8b29181e40f..00000000000 --- a/libs/dyn/mapping.go +++ /dev/null @@ -1,158 +0,0 @@ -package dyn - -import ( - "maps" - "slices" - - "github.com/databricks/cli/libs/diag" -) - -// Pair represents a single key-value pair in a Mapping. -type Pair struct { - Key Value - Value Value -} - -// Mapping represents a key-value map of dynamic values. -// It exists because plain Go maps cannot use dynamic values for keys. -// We need to use dynamic values for keys because it lets us associate metadata -// with keys (i.e. their definition location). Keys must be strings. -type Mapping struct { //nolint:recvcheck // value receivers for read-only accessors, pointer for mutators - pairs []Pair - index map[string]int -} - -// NewMapping creates a new empty Mapping. -func NewMapping() Mapping { - return Mapping{ - pairs: make([]Pair, 0), - index: make(map[string]int), - } -} - -// NewMappingFromPairs computes a [Mapping] from a list of [Pair]s. The index -// map does not need to be provided since that will be computed from the -// key-value pairs provided. -func NewMappingFromPairs(pairs []Pair) Mapping { - index := make(map[string]int) - for i, p := range pairs { - index[p.Key.MustString()] = i - } - - return Mapping{ - pairs: pairs, - index: index, - } -} - -// newMappingWithSize creates a new Mapping preallocated to the specified size. -func newMappingWithSize(size int) Mapping { - return Mapping{ - pairs: make([]Pair, 0, size), - index: make(map[string]int, size), - } -} - -// newMappingFromGoMap creates a new Mapping from a Go map of string keys and dynamic values. -func newMappingFromGoMap(vin map[string]Value) Mapping { - m := newMappingWithSize(len(vin)) - for k, v := range vin { - m.SetLoc(k, nil, v) - } - return m -} - -// Pairs returns all the key-value pairs in the Mapping. The pairs are sorted by -// their key in lexicographic order. -func (m Mapping) Pairs() []Pair { - return m.pairs -} - -// Len returns the number of key-value pairs in the Mapping. -func (m Mapping) Len() int { - return len(m.pairs) -} - -// GetPair returns the key-value pair with the specified key. -// It also returns a boolean indicating whether the pair was found. -func (m Mapping) GetPair(key Value) (Pair, bool) { - skey, ok := key.AsString() - if !ok { - return Pair{}, false - } - return m.GetPairByString(skey) -} - -// GetPairByString returns the key-value pair with the specified string key. -// It also returns a boolean indicating whether the pair was found. -func (m Mapping) GetPairByString(skey string) (Pair, bool) { - if i, ok := m.index[skey]; ok { - return m.pairs[i], true - } - return Pair{}, false -} - -// Get returns the value associated with the specified key. -// It also returns a boolean indicating whether the value was found. -func (m Mapping) Get(key Value) (Value, bool) { - p, ok := m.GetPair(key) - return p.Value, ok -} - -// GetByString returns the value associated with the specified string key. -// It also returns a boolean indicating whether the value was found. -func (m *Mapping) GetByString(skey string) (Value, bool) { - p, ok := m.GetPairByString(skey) - return p.Value, ok -} - -// Set sets the value for the given key in the mapping. -// If the key already exists, the value is updated. The location loc is ignored. -// If the key does not exist, a new key-value pair is added. -func (m *Mapping) SetLoc(skey string, loc []diag.Location, value Value) { - // If the key already exists, update the value. - if i, ok := m.index[skey]; ok { - m.pairs[i].Value = value - return - } - - // Otherwise, add a new pair. - m.pairs = append(m.pairs, Pair{NewValue(skey, loc), value}) - if m.index == nil { - m.index = make(map[string]int) - } - m.index[skey] = len(m.pairs) - 1 -} - -// Keys returns all the keys in the Mapping. -func (m Mapping) Keys() []Value { - keys := make([]Value, 0, len(m.pairs)) - for _, p := range m.pairs { - keys = append(keys, p.Key) - } - return keys -} - -// Values returns all the values in the Mapping. -func (m Mapping) Values() []Value { - values := make([]Value, 0, len(m.pairs)) - for _, p := range m.pairs { - values = append(values, p.Value) - } - return values -} - -// Clone creates a shallow copy of the Mapping. -func (m Mapping) Clone() Mapping { - return Mapping{ - pairs: slices.Clone(m.pairs), - index: maps.Clone(m.index), - } -} - -// Merge merges the key-value pairs from another Mapping into the current Mapping. -func (m *Mapping) Merge(n Mapping) { - for _, p := range n.pairs { - m.SetLoc(p.Key.MustString(), p.Key.Locations(), p.Value) - } -} diff --git a/libs/dyn/mapping_test.go b/libs/dyn/mapping_test.go deleted file mode 100644 index f646ab7c1d1..00000000000 --- a/libs/dyn/mapping_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package dyn_test - -import ( - "strconv" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestNewMapping(t *testing.T) { - m := dyn.NewMapping() - assert.Equal(t, 0, m.Len()) -} - -func TestMappingZeroValue(t *testing.T) { - var m dyn.Mapping - assert.Equal(t, 0, m.Len()) - - value, ok := m.Get(dyn.V("key")) - assert.Equal(t, dyn.InvalidValue, value) - assert.False(t, ok) - assert.Empty(t, m.Keys()) - assert.Empty(t, m.Values()) -} - -func TestMappingGet(t *testing.T) { - var m dyn.Mapping - m.SetLoc("key", nil, dyn.V("value")) - assert.Equal(t, 1, m.Len()) - - // Call GetPair - p, ok := m.GetPair(dyn.V("key")) - assert.True(t, ok) - assert.Equal(t, dyn.V("key"), p.Key) - assert.Equal(t, dyn.V("value"), p.Value) - - // Modify the value to make sure we're not getting a reference - p.Value = dyn.V("newvalue") - - // Call GetPair with invalid key - p, ok = m.GetPair(dyn.V(1234)) - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, p.Key) - assert.Equal(t, dyn.InvalidValue, p.Value) - - // Call GetPair with non-existent key - p, ok = m.GetPair(dyn.V("enoexist")) - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, p.Key) - assert.Equal(t, dyn.InvalidValue, p.Value) - - // Call GetPairByString - p, ok = m.GetPairByString("key") - assert.True(t, ok) - assert.Equal(t, dyn.V("key"), p.Key) - assert.Equal(t, dyn.V("value"), p.Value) - - // Modify the value to make sure we're not getting a reference - p.Value = dyn.V("newvalue") - - // Call GetPairByString with non-existent key - p, ok = m.GetPairByString("enoexist") - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, p.Key) - assert.Equal(t, dyn.InvalidValue, p.Value) - - // Call Get - value, ok := m.Get(dyn.V("key")) - assert.True(t, ok) - assert.Equal(t, dyn.V("value"), value) - - // Call Get with invalid key - value, ok = m.Get(dyn.V(1234)) - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, value) - - // Call Get with non-existent key - value, ok = m.Get(dyn.V("enoexist")) - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, value) - - // Call GetByString - value, ok = m.GetByString("key") - assert.True(t, ok) - assert.Equal(t, dyn.V("value"), value) - - // Call GetByString with non-existent key - value, ok = m.GetByString("enoexist") - assert.False(t, ok) - assert.Equal(t, dyn.InvalidValue, value) -} - -func TestMappingSetLoc(t *testing.T) { - var m dyn.Mapping - - // Set a value - m.SetLoc("key1", nil, dyn.V("foo")) - assert.Equal(t, 1, m.Len()) - - // Confirm the value - value, ok := m.GetByString("key1") - assert.True(t, ok) - assert.Equal(t, dyn.V("foo"), value) - - // Set another value - m.SetLoc("key2", nil, dyn.V("bar")) - assert.Equal(t, 2, m.Len()) - - // Confirm the value - value, ok = m.Get(dyn.V("key2")) - assert.True(t, ok) - assert.Equal(t, dyn.V("bar"), value) - - // Overwrite first value - m.SetLoc("key1", nil, dyn.V("qux")) - assert.Equal(t, 2, m.Len()) - - // Confirm the value - value, ok = m.Get(dyn.V("key1")) - assert.True(t, ok) - assert.Equal(t, dyn.V("qux"), value) -} - -func TestMappingKeysValues(t *testing.T) { - // Configure mapping - var m dyn.Mapping - m.SetLoc("key1", nil, dyn.V("foo")) - m.SetLoc("key2", nil, dyn.V("bar")) - - // Confirm keys - keys := m.Keys() - assert.Len(t, keys, 2) - assert.Contains(t, keys, dyn.V("key1")) - assert.Contains(t, keys, dyn.V("key2")) - - // Confirm values - values := m.Values() - assert.Len(t, values, 2) - assert.Contains(t, values, dyn.V("foo")) - assert.Contains(t, values, dyn.V("bar")) -} - -func TestMappingClone(t *testing.T) { - // Configure mapping - var m1 dyn.Mapping - m1.SetLoc("key1", nil, dyn.V("foo")) - m1.SetLoc("key2", nil, dyn.V("bar")) - - // Clone mapping - m2 := m1.Clone() - assert.Equal(t, m1.Len(), m2.Len()) - - // Modify original mapping - m1.SetLoc("key1", nil, dyn.V("qux")) - - // Confirm values - value, ok := m1.Get(dyn.V("key1")) - assert.True(t, ok) - assert.Equal(t, dyn.V("qux"), value) - value, ok = m2.Get(dyn.V("key1")) - assert.True(t, ok) - assert.Equal(t, dyn.V("foo"), value) -} - -func TestMappingMerge(t *testing.T) { - var m1 dyn.Mapping - for i := range 10 { - m1.SetLoc(strconv.Itoa(i), nil, dyn.V(i)) - } - - var m2 dyn.Mapping - for i := 5; i < 15; i++ { - m2.SetLoc(strconv.Itoa(i), nil, dyn.V(i)) - } - - var out dyn.Mapping - out.Merge(m1) - assert.Equal(t, 10, out.Len()) - out.Merge(m2) - assert.Equal(t, 15, out.Len()) -} diff --git a/libs/dyn/merge/elements_by_key.go b/libs/dyn/merge/elements_by_key.go deleted file mode 100644 index 7d7e35c8eb5..00000000000 --- a/libs/dyn/merge/elements_by_key.go +++ /dev/null @@ -1,103 +0,0 @@ -package merge - -import ( - "slices" - - "github.com/databricks/cli/libs/dyn" -) - -type elementsByKey struct { - key string - keyFunc func(dyn.Value) string - sortKeys bool -} - -func (e elementsByKey) doMap(_ dyn.Path, v dyn.Value, mergeFunc func(a, b dyn.Value) (dyn.Value, error)) (dyn.Value, error) { - // We know the type of this value is a sequence. - // For additional defence, return self if it is not. - elements, ok := v.AsSequence() - if !ok { - return v, nil - } - - seen := make(map[string]dyn.Value, len(elements)) - keys := make([]string, 0, len(elements)) - - // Iterate in natural order. For a given key, we first see the - // base definition and merge instances that come after it. - for i := range elements { - kv := elements[i].Get(e.key) - key := e.keyFunc(kv) - - // Register element with key if not yet seen before. - ref, ok := seen[key] - if !ok { - keys = append(keys, key) - seen[key] = elements[i] - continue - } - - // Merge this instance into the reference. - nv, err := mergeFunc(ref, elements[i]) - if err != nil { - return v, err - } - - // Overwrite reference. - seen[key] = nv - } - - if e.sortKeys { - slices.Sort(keys) - } - - // Gather resulting elements in natural order. - out := make([]dyn.Value, 0, len(keys)) - for _, key := range keys { - // Preserve the location from the original key value so that - // downstream code can tell this field was defined in the YAML. - keyLoc := seen[key].Get(e.key).Locations() - nv, err := dyn.Set(seen[key], e.key, dyn.NewValue(key, keyLoc)) - if err != nil { - return dyn.InvalidValue, err - } - out = append(out, nv) - } - - return dyn.NewValue(out, v.Locations()), nil -} - -func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return e.doMap(nil, v, Merge) -} - -func (e elementsByKey) MapWithOverride(p dyn.Path, v dyn.Value) (dyn.Value, error) { - return e.doMap(nil, v, func(a, b dyn.Value) (dyn.Value, error) { - return Override(a, b, OverrideVisitor{ - VisitInsert: func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return v, nil - }, - VisitDelete: func(valuePath dyn.Path, left dyn.Value) error { - return nil - }, - VisitUpdate: func(_ dyn.Path, a, b dyn.Value) (dyn.Value, error) { - return b, nil - }, - }) - }) -} - -// ElementsByKey returns a [dyn.MapFunc] that operates on a sequence -// where each element is a map. It groups elements by a key and merges -// elements with the same key. -// -// The function that extracts the key from an element is provided as -// a parameter. The resulting elements get their key field overwritten -// with the value as returned by the key function. -func ElementsByKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc { - return elementsByKey{key, keyFunc, false}.Map -} - -func ElementsByKeyWithOverride(key string, keyFunc func(dyn.Value) string) dyn.MapFunc { - return elementsByKey{key, keyFunc, false}.MapWithOverride -} diff --git a/libs/dyn/merge/elements_by_key_test.go b/libs/dyn/merge/elements_by_key_test.go deleted file mode 100644 index 2aa19306820..00000000000 --- a/libs/dyn/merge/elements_by_key_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package merge - -import ( - "strings" - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestElementByKey(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "key": dyn.V("foo"), - "value": dyn.V(42), - }), - dyn.V(map[string]dyn.Value{ - "key": dyn.V("bar"), - "value": dyn.V(43), - }), - dyn.V(map[string]dyn.Value{ - // Use upper case key to test that the resulting element has its - // key field assigned to the output of the key function. - // The key function in this test returns the lower case version of the key. - "key": dyn.V("FOO"), - "value": dyn.V(44), - }), - }) - - keyFunc := func(v dyn.Value) string { - return strings.ToLower(v.MustString()) - } - - vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKey("key", keyFunc)) - require.NoError(t, err) - assert.Len(t, vout.MustSequence(), 2) - assert.Equal(t, - map[string]any{ - "key": "foo", - "value": 44, - }, vout.Index(0).AsAny(), - ) - assert.Equal(t, - map[string]any{ - "key": "bar", - "value": 43, - }, vout.Index(1).AsAny(), - ) -} - -func TestElementByKeyPreservesLocations(t *testing.T) { - loc := diag.Location{File: "config.yml", Line: 10, Column: 5} - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "key": dyn.NewValue("foo", []diag.Location{loc}), - "value": dyn.V(42), - }), - }) - - keyFunc := func(v dyn.Value) string { - return v.MustString() - } - - vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKey("key", keyFunc)) - require.NoError(t, err) - - // Verify the key field retains its original location. - keyValue := vout.Index(0).Get("key") - assert.Equal(t, loc, keyValue.Location()) -} - -func TestElementByKeyWithOverride(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "key": dyn.V("foo"), - "value": dyn.V(42), - }), - dyn.V(map[string]dyn.Value{ - "key": dyn.V("bar"), - "value": dyn.V(43), - }), - dyn.V(map[string]dyn.Value{ - "key": dyn.V("foo"), - "othervalue": dyn.V(44), - }), - }) - - keyFunc := func(v dyn.Value) string { - return strings.ToLower(v.MustString()) - } - - vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKeyWithOverride("key", keyFunc)) - require.NoError(t, err) - assert.Len(t, vout.MustSequence(), 2) - assert.Equal(t, - map[string]any{ - "key": "foo", - "othervalue": 44, - }, vout.Index(0).AsAny(), - ) - assert.Equal(t, - map[string]any{ - "key": "bar", - "value": 43, - }, vout.Index(1).AsAny(), - ) -} diff --git a/libs/dyn/merge/merge.go b/libs/dyn/merge/merge.go deleted file mode 100644 index 374450ef240..00000000000 --- a/libs/dyn/merge/merge.go +++ /dev/null @@ -1,119 +0,0 @@ -package merge - -import ( - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -// Merge recursively merges the specified values. -// -// Semantics are as follows: -// * Merging x with nil or nil with x always yields x. -// * Merging maps a and b means entries from map b take precedence. -// * Merging sequences a and b means concatenating them. -// -// Merging retains and accumulates the locations metadata associated with the values. -// This allows users of the module to track the provenance of values across merging of -// configuration trees, which is useful for reporting errors and warnings. -// -// Semantics for location metadata in the merged value are similar to the semantics -// for the values themselves: -// -// - When merging x with nil or nil with x, the location of x is retained. -// -// - When merging maps or sequences, the combined value retains the location of a and -// accumulates the location of b. The individual elements of the map or sequence retain -// their original locations, i.e., whether they were originally defined in a or b. -// -// The rationale for retaining location of a is that we would like to return -// the first location a bit of configuration showed up when reporting errors and warnings. -// -// - Merging primitive values means using the incoming value `b`. The location of the -// incoming value is retained and the location of the existing value `a` is accumulated. -// This is because the incoming value overwrites the existing value. -func Merge(a, b dyn.Value) (dyn.Value, error) { - return merge(a, b) -} - -func merge(a, b dyn.Value) (dyn.Value, error) { - ak := a.Kind() - bk := b.Kind() - - // If a is nil, return b. - if ak == dyn.KindNil { - return b.AppendLocationsFromValue(a), nil - } - - // If b is nil, return a. - if bk == dyn.KindNil { - return a.AppendLocationsFromValue(b), nil - } - - // Call the appropriate merge function based on the kind of a and b. - switch ak { - case dyn.KindMap: - if bk != dyn.KindMap { - return dyn.InvalidValue, fmt.Errorf("cannot merge map with %s", bk) - } - return mergeMap(a, b) - case dyn.KindSequence: - if bk != dyn.KindSequence { - return dyn.InvalidValue, fmt.Errorf("cannot merge sequence with %s", bk) - } - return mergeSequence(a, b) - default: - if ak != bk { - return dyn.InvalidValue, fmt.Errorf("cannot merge %s with %s", ak, bk) - } - return mergePrimitive(a, b) - } -} - -func mergeMap(a, b dyn.Value) (dyn.Value, error) { - out := dyn.NewMapping() - am := a.MustMap() - bm := b.MustMap() - - // Add the values from a into the output map. - out.Merge(am) - - // Merge the values from b into the output map. - for _, pair := range bm.Pairs() { - pk := pair.Key - key := pk.MustString() - pv := pair.Value - if ov, ok := out.Get(pk); ok { - // If the key already exists, merge the values. - merged, err := merge(ov, pv) - if err != nil { - return dyn.InvalidValue, err - } - out.SetLoc(key, pair.Key.Locations(), merged) - } else { - // Otherwise, just set the value. - out.SetLoc(key, pair.Key.Locations(), pv) - } - } - - // Preserve the location of the first value. Accumulate the locations of the second value. - return dyn.NewValue(out, a.Locations()).AppendLocationsFromValue(b), nil -} - -func mergeSequence(a, b dyn.Value) (dyn.Value, error) { - as := a.MustSequence() - bs := b.MustSequence() - - // Merging sequences means concatenating them. - out := make([]dyn.Value, len(as)+len(bs)) - copy(out[:], as) - copy(out[len(as):], bs) - - // Preserve the location of the first value. Accumulate the locations of the second value. - return dyn.NewValue(out, a.Locations()).AppendLocationsFromValue(b), nil -} - -func mergePrimitive(a, b dyn.Value) (dyn.Value, error) { - // Merging primitive values means using the incoming value. - return b.AppendLocationsFromValue(a), nil -} diff --git a/libs/dyn/merge/merge_test.go b/libs/dyn/merge/merge_test.go deleted file mode 100644 index 9545c752258..00000000000 --- a/libs/dyn/merge/merge_test.go +++ /dev/null @@ -1,301 +0,0 @@ -package merge - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestMergeMaps(t *testing.T) { - l1 := diag.Location{File: "file1", Line: 1, Column: 2} - v1 := dyn.NewValue(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", []diag.Location{l1}), - "bar": dyn.NewValue("baz", []diag.Location{l1}), - }, []diag.Location{l1}) - - l2 := diag.Location{File: "file2", Line: 3, Column: 4} - v2 := dyn.NewValue(map[string]dyn.Value{ - "bar": dyn.NewValue("qux", []diag.Location{l2}), - "qux": dyn.NewValue("foo", []diag.Location{l2}), - }, []diag.Location{l2}) - - // Merge v2 into v1. - { - out, err := Merge(v1, v2) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": "bar", - "bar": "qux", - "qux": "foo", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l1, l2}, out.Locations()) - assert.Equal(t, []diag.Location{l2, l1}, out.Get("bar").Locations()) - assert.Equal(t, []diag.Location{l1}, out.Get("foo").Locations()) - assert.Equal(t, []diag.Location{l2}, out.Get("qux").Locations()) - - // Location of the merged value should be the location of v1. - assert.Equal(t, l1, out.Location()) - - // Value of bar is "qux" which comes from v2. This .Location() should - // return the location of v2. - assert.Equal(t, l2, out.Get("bar").Location()) - - // Original locations of keys that were not overwritten should be preserved. - assert.Equal(t, l1, out.Get("foo").Location()) - assert.Equal(t, l2, out.Get("qux").Location()) - } - - // Merge v1 into v2. - { - out, err := Merge(v2, v1) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": "bar", - "bar": "baz", - "qux": "foo", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l2, l1}, out.Locations()) - assert.Equal(t, []diag.Location{l1, l2}, out.Get("bar").Locations()) - assert.Equal(t, []diag.Location{l1}, out.Get("foo").Locations()) - assert.Equal(t, []diag.Location{l2}, out.Get("qux").Locations()) - - // Location of the merged value should be the location of v2. - assert.Equal(t, l2, out.Location()) - - // Value of bar is "baz" which comes from v1. This .Location() should - // return the location of v1. - assert.Equal(t, l1, out.Get("bar").Location()) - - // Original locations of keys that were not overwritten should be preserved. - assert.Equal(t, l1, out.Get("foo").Location()) - assert.Equal(t, l2, out.Get("qux").Location()) - } -} - -func TestMergeMapsNil(t *testing.T) { - l := diag.Location{File: "file", Line: 1, Column: 2} - v := dyn.NewValue(map[string]dyn.Value{ - "foo": dyn.V("bar"), - }, []diag.Location{l}) - - nilL := diag.Location{File: "file", Line: 3, Column: 4} - nilV := dyn.NewValue(nil, []diag.Location{nilL}) - - // Merge nil into v. - { - out, err := Merge(v, nilV) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": "bar", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l, nilL}, out.Locations()) - - // Location of the non-nil value should be returned by .Location(). - assert.Equal(t, l, out.Location()) - } - - // Merge v into nil. - { - out, err := Merge(nilV, v) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": "bar", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l, nilL}, out.Locations()) - - // Location of the non-nil value should be returned by .Location(). - assert.Equal(t, l, out.Location()) - } -} - -func TestMergeMapsError(t *testing.T) { - v := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - }) - - other := dyn.V("string") - - // Merge a string into v. - { - out, err := Merge(v, other) - assert.EqualError(t, err, "cannot merge map with string") - assert.Equal(t, dyn.InvalidValue, out) - } -} - -func TestMergeSequences(t *testing.T) { - l1 := diag.Location{File: "file1", Line: 1, Column: 2} - v1 := dyn.NewValue([]dyn.Value{ - dyn.NewValue("bar", []diag.Location{l1}), - dyn.NewValue("baz", []diag.Location{l1}), - }, []diag.Location{l1}) - - l2 := diag.Location{File: "file2", Line: 3, Column: 4} - l3 := diag.Location{File: "file3", Line: 5, Column: 6} - v2 := dyn.NewValue([]dyn.Value{ - dyn.NewValue("qux", []diag.Location{l2}), - dyn.NewValue("foo", []diag.Location{l3}), - }, []diag.Location{l2, l3}) - - // Merge v2 into v1. - { - out, err := Merge(v1, v2) - assert.NoError(t, err) - assert.Equal(t, []any{ - "bar", - "baz", - "qux", - "foo", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l1, l2, l3}, out.Locations()) - - // Location of the merged value should be the location of v1. - assert.Equal(t, l1, out.Location()) - - // Location of the individual values should be preserved. - assert.Equal(t, l1, out.Index(0).Location()) // "bar" - assert.Equal(t, l1, out.Index(1).Location()) // "baz" - assert.Equal(t, l2, out.Index(2).Location()) // "qux" - assert.Equal(t, l3, out.Index(3).Location()) // "foo" - } - - // Merge v1 into v2. - { - out, err := Merge(v2, v1) - assert.NoError(t, err) - assert.Equal(t, []any{ - "qux", - "foo", - "bar", - "baz", - }, out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l2, l3, l1}, out.Locations()) - - // Location of the merged value should be the location of v2. - assert.Equal(t, l2, out.Location()) - - // Location of the individual values should be preserved. - assert.Equal(t, l2, out.Index(0).Location()) // "qux" - assert.Equal(t, l3, out.Index(1).Location()) // "foo" - assert.Equal(t, l1, out.Index(2).Location()) // "bar" - assert.Equal(t, l1, out.Index(3).Location()) // "baz" - } -} - -func TestMergeSequencesNil(t *testing.T) { - v := dyn.V([]dyn.Value{ - dyn.V("bar"), - }) - - // Merge nil into v. - { - out, err := Merge(v, dyn.NilValue) - assert.NoError(t, err) - assert.Equal(t, []any{ - "bar", - }, out.AsAny()) - } - - // Merge v into nil. - { - out, err := Merge(dyn.NilValue, v) - assert.NoError(t, err) - assert.Equal(t, []any{ - "bar", - }, out.AsAny()) - } -} - -func TestMergeSequencesError(t *testing.T) { - v := dyn.V([]dyn.Value{ - dyn.V("bar"), - }) - - other := dyn.V("string") - - // Merge a string into v. - { - out, err := Merge(v, other) - assert.EqualError(t, err, "cannot merge sequence with string") - assert.Equal(t, dyn.InvalidValue, out) - } -} - -func TestMergePrimitives(t *testing.T) { - l1 := diag.Location{File: "file1", Line: 1, Column: 2} - l2 := diag.Location{File: "file2", Line: 3, Column: 4} - v1 := dyn.NewValue("bar", []diag.Location{l1}) - v2 := dyn.NewValue("baz", []diag.Location{l2}) - - // Merge v2 into v1. - { - out, err := Merge(v1, v2) - assert.NoError(t, err) - assert.Equal(t, "baz", out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l2, l1}, out.Locations()) - - // Location of the merged value should be the location of v2, the second value. - assert.Equal(t, l2, out.Location()) - } - - // Merge v1 into v2. - { - out, err := Merge(v2, v1) - assert.NoError(t, err) - assert.Equal(t, "bar", out.AsAny()) - - // Locations of both values should be preserved. - assert.Equal(t, []diag.Location{l1, l2}, out.Locations()) - - // Location of the merged value should be the location of v1, the second value. - assert.Equal(t, l1, out.Location()) - } -} - -func TestMergePrimitivesNil(t *testing.T) { - v := dyn.V("bar") - - // Merge nil into v. - { - out, err := Merge(v, dyn.NilValue) - assert.NoError(t, err) - assert.Equal(t, "bar", out.AsAny()) - } - - // Merge v into nil. - { - out, err := Merge(dyn.NilValue, v) - assert.NoError(t, err) - assert.Equal(t, "bar", out.AsAny()) - } -} - -func TestMergePrimitivesError(t *testing.T) { - v := dyn.V("bar") - other := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - }) - - // Merge a map into v. - { - out, err := Merge(v, other) - assert.EqualError(t, err, "cannot merge string with map") - assert.Equal(t, dyn.InvalidValue, out) - } -} diff --git a/libs/dyn/merge/override.go b/libs/dyn/merge/override.go deleted file mode 100644 index 873c38eead1..00000000000 --- a/libs/dyn/merge/override.go +++ /dev/null @@ -1,200 +0,0 @@ -package merge - -import ( - "errors" - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -// OverrideVisitor is visiting the changes during the override process -// and allows to control what changes are allowed, or update the effective -// value. -// -// For instance, it can disallow changes outside the specific path(s), or update -// the location of the effective value. -// -// Values returned by 'VisitInsert' and 'VisitUpdate' are used as the final value -// of the node. 'VisitDelete' can return ErrOverrideUndoDelete to undo delete. -// -// 'VisitDelete' is called when a value is removed from mapping or sequence -// 'VisitInsert' is called when a new value is added to mapping or sequence -// 'VisitUpdate' is called when a leaf value is updated -type OverrideVisitor struct { - VisitDelete func(valuePath dyn.Path, left dyn.Value) error - VisitInsert func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) - VisitUpdate func(valuePath dyn.Path, left, right dyn.Value) (dyn.Value, error) -} - -var ErrOverrideUndoDelete = errors.New("undo delete operation") - -// Override overrides value 'leftRoot' with 'rightRoot', keeping 'location' if values -// haven't changed. Preserving 'location' is important to preserve the original source of the value -// for error reporting. -func Override(leftRoot, rightRoot dyn.Value, visitor OverrideVisitor) (dyn.Value, error) { - return override(dyn.EmptyPath, leftRoot, rightRoot, visitor) -} - -func override(basePath dyn.Path, left, right dyn.Value, visitor OverrideVisitor) (dyn.Value, error) { - if left.Kind() != right.Kind() { - return visitor.VisitUpdate(basePath, left, right) - } - - // NB: we only call 'VisitUpdate' on leaf values, and for sequences and mappings - // we don't know if value was updated or not - - switch left.Kind() { - case dyn.KindMap: - merged, err := overrideMapping(basePath, left.MustMap(), right.MustMap(), visitor) - if err != nil { - return dyn.InvalidValue, err - } - - return dyn.NewValue(merged, left.Locations()), nil - - case dyn.KindSequence: - // some sequences are keyed, and we can detect which elements are added/removed/updated, - // but we don't have this information - merged, err := overrideSequence(basePath, left.MustSequence(), right.MustSequence(), visitor) - if err != nil { - return dyn.InvalidValue, err - } - - return dyn.NewValue(merged, left.Locations()), nil - - case dyn.KindString: - if left.MustString() == right.MustString() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindFloat: - // TODO consider comparison with epsilon if normalization doesn't help, where do we use floats? - - if left.MustFloat() == right.MustFloat() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindBool: - if left.MustBool() == right.MustBool() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindTime: - if left.MustTime() == right.MustTime() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - - case dyn.KindInt: - if left.MustInt() == right.MustInt() { - return left, nil - } else { - return visitor.VisitUpdate(basePath, left, right) - } - case dyn.KindNil: - return left, nil - default: - // Fall through to the error case. - } - - return dyn.InvalidValue, fmt.Errorf("unexpected kind %s at %s", left.Kind(), basePath.String()) -} - -func overrideMapping(basePath dyn.Path, leftMapping, rightMapping dyn.Mapping, visitor OverrideVisitor) (dyn.Mapping, error) { - out := dyn.NewMapping() - - for _, leftPair := range leftMapping.Pairs() { - // detect if key was removed - if _, ok := rightMapping.GetPair(leftPair.Key); !ok { - key := leftPair.Key.MustString() - keyLoc := leftPair.Key.Locations() - path := basePath.Append(dyn.Key(key)) - - err := visitor.VisitDelete(path, leftPair.Value) - - // if 'delete' was undone, add it back - if errors.Is(err, ErrOverrideUndoDelete) { - out.SetLoc(key, keyLoc, leftPair.Value) - } else if err != nil { - return dyn.NewMapping(), err - } - } - } - - // iterating only right mapping will remove keys not present anymore - // and insert new keys - - for _, rightPair := range rightMapping.Pairs() { - key := rightPair.Key.MustString() - keyLoc := rightPair.Key.Locations() - if leftPair, ok := leftMapping.GetPair(rightPair.Key); ok { - path := basePath.Append(dyn.Key(key)) - newValue, err := override(path, leftPair.Value, rightPair.Value, visitor) - if err != nil { - return dyn.NewMapping(), err - } - - // key was there before, so keep its location - out.SetLoc(key, keyLoc, newValue) - } else { - path := basePath.Append(dyn.Key(rightPair.Key.MustString())) - - newValue, err := visitor.VisitInsert(path, rightPair.Value) - if err != nil { - return dyn.NewMapping(), err - } - - out.SetLoc(key, keyLoc, newValue) - } - } - - return out, nil -} - -func overrideSequence(basePath dyn.Path, left, right []dyn.Value, visitor OverrideVisitor) ([]dyn.Value, error) { - minLen := min(len(left), len(right)) - var values []dyn.Value - - for i := range minLen { - path := basePath.Append(dyn.Index(i)) - merged, err := override(path, left[i], right[i], visitor) - if err != nil { - return nil, err - } - - values = append(values, merged) - } - - if len(right) > len(left) { - for i := minLen; i < len(right); i++ { - path := basePath.Append(dyn.Index(i)) - newValue, err := visitor.VisitInsert(path, right[i]) - if err != nil { - return nil, err - } - - values = append(values, newValue) - } - } else if len(left) > len(right) { - for i := minLen; i < len(left); i++ { - path := basePath.Append(dyn.Index(i)) - err := visitor.VisitDelete(path, left[i]) - - // if 'delete' was undone, add it back - if errors.Is(err, ErrOverrideUndoDelete) { - values = append(values, left[i]) - } else if err != nil { - return nil, err - } - } - } - - return values, nil -} diff --git a/libs/dyn/merge/override_test.go b/libs/dyn/merge/override_test.go deleted file mode 100644 index 17602f57ad7..00000000000 --- a/libs/dyn/merge/override_test.go +++ /dev/null @@ -1,520 +0,0 @@ -package merge - -import ( - "errors" - "testing" - "time" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type overrideTestCase struct { - name string - left dyn.Value - right dyn.Value - state visitorState - expected dyn.Value -} - -func TestOverride_Primitive(t *testing.T) { - leftLocation := diag.Location{File: "left.yml", Line: 1, Column: 1} - rightLocation := diag.Location{File: "right.yml", Line: 1, Column: 1} - - modifiedTestCases := []overrideTestCase{ - { - name: "string (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue("a", []diag.Location{leftLocation}), - right: dyn.NewValue("b", []diag.Location{rightLocation}), - expected: dyn.NewValue("b", []diag.Location{rightLocation}), - }, - { - name: "string (not updated)", - state: visitorState{}, - left: dyn.NewValue("a", []diag.Location{leftLocation}), - right: dyn.NewValue("a", []diag.Location{rightLocation}), - expected: dyn.NewValue("a", []diag.Location{leftLocation}), - }, - { - name: "bool (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(true, []diag.Location{leftLocation}), - right: dyn.NewValue(false, []diag.Location{rightLocation}), - expected: dyn.NewValue(false, []diag.Location{rightLocation}), - }, - { - name: "bool (not updated)", - state: visitorState{}, - left: dyn.NewValue(true, []diag.Location{leftLocation}), - right: dyn.NewValue(true, []diag.Location{rightLocation}), - expected: dyn.NewValue(true, []diag.Location{leftLocation}), - }, - { - name: "int (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(1, []diag.Location{leftLocation}), - right: dyn.NewValue(2, []diag.Location{rightLocation}), - expected: dyn.NewValue(2, []diag.Location{rightLocation}), - }, - { - name: "int (not updated)", - state: visitorState{}, - left: dyn.NewValue(int32(1), []diag.Location{leftLocation}), - right: dyn.NewValue(int64(1), []diag.Location{rightLocation}), - expected: dyn.NewValue(int32(1), []diag.Location{leftLocation}), - }, - { - name: "float (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(1.0, []diag.Location{leftLocation}), - right: dyn.NewValue(2.0, []diag.Location{rightLocation}), - expected: dyn.NewValue(2.0, []diag.Location{rightLocation}), - }, - { - name: "float (not updated)", - state: visitorState{}, - left: dyn.NewValue(float32(1.0), []diag.Location{leftLocation}), - right: dyn.NewValue(float64(1.0), []diag.Location{rightLocation}), - expected: dyn.NewValue(float32(1.0), []diag.Location{leftLocation}), - }, - { - name: "time (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []diag.Location{leftLocation}), - right: dyn.NewValue(dyn.FromTime(time.UnixMilli(10001)), []diag.Location{rightLocation}), - expected: dyn.NewValue(dyn.FromTime(time.UnixMilli(10001)), []diag.Location{rightLocation}), - }, - { - name: "time (not updated)", - state: visitorState{}, - left: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []diag.Location{leftLocation}), - right: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []diag.Location{rightLocation}), - expected: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []diag.Location{leftLocation}), - }, - { - name: "different types (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue("a", []diag.Location{leftLocation}), - right: dyn.NewValue(42, []diag.Location{rightLocation}), - expected: dyn.NewValue(42, []diag.Location{rightLocation}), - }, - { - name: "map - remove 'a', update 'b'", - state: visitorState{ - removed: []string{"root.a"}, - updated: []string{"root.b"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []diag.Location{leftLocation}), - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}), - - right: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(20, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}), - - expected: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(20, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}), - }, - { - name: "map - add 'a'", - state: visitorState{ - added: []string{"root.a"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - - right: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []diag.Location{rightLocation}), - "b": dyn.NewValue(10, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - - expected: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []diag.Location{rightLocation}), - // location hasn't changed because value hasn't changed - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "map - remove 'a'", - state: visitorState{ - removed: []string{"root.a"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "a": dyn.NewValue(42, []diag.Location{leftLocation}), - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - - right: dyn.NewValue( - map[string]dyn.Value{ - "b": dyn.NewValue(10, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - - expected: dyn.NewValue( - map[string]dyn.Value{ - // location hasn't changed because value hasn't changed - "b": dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "map - add 'jobs.job_1'", - state: visitorState{ - added: []string{"root.jobs.job_1"}, - }, - left: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - []diag.Location{leftLocation}, - ), - - right: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{rightLocation}), - "job_1": dyn.NewValue(1337, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - }, - []diag.Location{rightLocation}, - ), - - expected: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{leftLocation}), - "job_1": dyn.NewValue(1337, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "map - remove nested key", - state: visitorState{removed: []string{"root.jobs.job_1"}}, - left: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{leftLocation}), - "job_1": dyn.NewValue(1337, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - []diag.Location{leftLocation}, - ), - right: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - }, - []diag.Location{rightLocation}, - ), - expected: dyn.NewValue( - map[string]dyn.Value{ - "jobs": dyn.NewValue( - map[string]dyn.Value{ - "job_0": dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "sequence - add", - state: visitorState{added: []string{"root[1]"}}, - left: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - right: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{rightLocation}), - dyn.NewValue(10, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - expected: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - dyn.NewValue(10, []diag.Location{rightLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "sequence - remove", - state: visitorState{removed: []string{"root[1]"}}, - left: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - dyn.NewValue(10, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - right: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - expected: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - // location hasn't changed because value hasn't changed - }, - { - name: "sequence (not updated)", - state: visitorState{}, - left: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - right: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{rightLocation}), - }, - []diag.Location{rightLocation}, - ), - expected: dyn.NewValue( - []dyn.Value{ - dyn.NewValue(42, []diag.Location{leftLocation}), - }, - []diag.Location{leftLocation}, - ), - }, - { - name: "nil (not updated)", - state: visitorState{}, - left: dyn.NilValue.WithLocations([]diag.Location{leftLocation}), - right: dyn.NilValue.WithLocations([]diag.Location{rightLocation}), - expected: dyn.NilValue.WithLocations([]diag.Location{leftLocation}), - }, - { - name: "nil (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NilValue, - right: dyn.NewValue(42, []diag.Location{rightLocation}), - expected: dyn.NewValue(42, []diag.Location{rightLocation}), - }, - { - name: "change kind (updated)", - state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(42.0, []diag.Location{leftLocation}), - right: dyn.NewValue(42, []diag.Location{rightLocation}), - expected: dyn.NewValue(42, []diag.Location{rightLocation}), - }, - } - - for _, tc := range modifiedTestCases { - t.Run(tc.name, func(t *testing.T) { - s, visitor := createVisitor(visitorOpts{}) - out, err := override(dyn.NewPath(dyn.Key("root")), tc.left, tc.right, visitor) - - assert.NoError(t, err) - dynassert.Equal(t, tc.state, *s) - dynassert.Equal(t, tc.expected, out) - }) - - modified := len(tc.state.removed)+len(tc.state.added)+len(tc.state.updated) > 0 - - // visitor is not used unless there is a change - - if modified { - t.Run(tc.name+" - visitor has error", func(t *testing.T) { - _, visitor := createVisitor(visitorOpts{error: errors.New("unexpected change in test")}) - _, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) - - assert.EqualError(t, err, "unexpected change in test") - }) - - t.Run(tc.name+" - visitor overrides value", func(t *testing.T) { - expected := dyn.V("return value") - s, visitor := createVisitor(visitorOpts{returnValue: &expected}) - out, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) - - assert.NoError(t, err) - - for _, added := range s.added { - actual, err := dyn.GetByPath(out, dyn.MustPathFromString(added)) - - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) - } - - for _, updated := range s.updated { - actual, err := dyn.GetByPath(out, dyn.MustPathFromString(updated)) - - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) - } - }) - - if len(tc.state.removed) > 0 { - t.Run(tc.name+" - visitor can undo delete", func(t *testing.T) { - s, visitor := createVisitor(visitorOpts{deleteError: ErrOverrideUndoDelete}) - out, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) - require.NoError(t, err) - - for _, removed := range s.removed { - expected, err := dyn.GetByPath(tc.left, dyn.MustPathFromString(removed)) - require.NoError(t, err) - - actual, err := dyn.GetByPath(out, dyn.MustPathFromString(removed)) - - assert.NoError(t, err) - dynassert.Equal(t, expected, actual) - } - }) - } - } - } -} - -func TestOverride_PreserveMappingKeys(t *testing.T) { - leftLocation := diag.Location{File: "left.yml", Line: 1, Column: 1} - leftKeyLocation := diag.Location{File: "left.yml", Line: 2, Column: 1} - leftValueLocation := diag.Location{File: "left.yml", Line: 3, Column: 1} - - rightLocation := diag.Location{File: "right.yml", Line: 1, Column: 1} - rightKeyLocation := diag.Location{File: "right.yml", Line: 2, Column: 1} - rightValueLocation := diag.Location{File: "right.yml", Line: 3, Column: 1} - - left := dyn.NewMapping() - left.SetLoc("a", []diag.Location{leftKeyLocation}, dyn.NewValue(42, []diag.Location{leftValueLocation})) - - right := dyn.NewMapping() - right.SetLoc("a", []diag.Location{rightKeyLocation}, dyn.NewValue(7, []diag.Location{rightValueLocation})) - - state, visitor := createVisitor(visitorOpts{}) - - out, err := override( - dyn.EmptyPath, - dyn.NewValue(left, []diag.Location{leftLocation}), - dyn.NewValue(right, []diag.Location{rightLocation}), - visitor, - ) - - assert.NoError(t, err) - - if err != nil { - outPairs := out.MustMap().Pairs() - - assert.Equal(t, visitorState{updated: []string{"a"}}, state) - assert.Len(t, outPairs, 1) - - // mapping was first defined in left, so it should keep its location - assert.Equal(t, leftLocation, out.Location()) - - // if there is a validation error for key value, it should point - // to where it was initially defined - assert.Equal(t, leftKeyLocation, outPairs[0].Key.Location()) - - // the value should have updated location, because it has changed - assert.Equal(t, rightValueLocation, outPairs[0].Value.Location()) - } -} - -type visitorState struct { - added []string - removed []string - updated []string -} - -type visitorOpts struct { - error error - deleteError error - returnValue *dyn.Value -} - -func createVisitor(opts visitorOpts) (*visitorState, OverrideVisitor) { - s := visitorState{} - - return &s, OverrideVisitor{ - VisitUpdate: func(valuePath dyn.Path, left, right dyn.Value) (dyn.Value, error) { - s.updated = append(s.updated, valuePath.String()) - - if opts.error != nil { - return dyn.NilValue, opts.error - } else if opts.returnValue != nil { - return *opts.returnValue, nil - } else { - return right, nil - } - }, - VisitDelete: func(valuePath dyn.Path, left dyn.Value) error { - s.removed = append(s.removed, valuePath.String()) - - if opts.error != nil { - return opts.error - } else if opts.deleteError != nil { - return opts.deleteError - } else { - return nil - } - }, - VisitInsert: func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) { - s.added = append(s.added, valuePath.String()) - - if opts.error != nil { - return dyn.NilValue, opts.error - } else if opts.returnValue != nil { - return *opts.returnValue, nil - } else { - return right, nil - } - }, - } -} diff --git a/libs/dyn/merge/select.go b/libs/dyn/merge/select.go deleted file mode 100644 index 9b09bfa2162..00000000000 --- a/libs/dyn/merge/select.go +++ /dev/null @@ -1,53 +0,0 @@ -package merge - -import ( - "fmt" - - "github.com/databricks/cli/libs/dyn" -) - -// Select returns a new map that contains only the keys specified in the included list. -func Select(value dyn.Value, included []string) (dyn.Value, error) { - mapping, ok := value.AsMap() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected a map, but found %s", value.Kind()) - } - - newMapping := dyn.NewMapping() - for _, key := range included { - pair, ok := mapping.GetPairByString(key) - - if ok { - newMapping.SetLoc(key, pair.Key.Locations(), pair.Value) - } - } - - return dyn.NewValue(newMapping, value.Locations()), nil -} - -// AntiSelect returns a new map with all keys from the input map except for the ones in the excluded list. -func AntiSelect(value dyn.Value, excluded []string) (dyn.Value, error) { - mapping, ok := value.AsMap() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected a map, but found %s", value.Kind()) - } - - excludedSet := make(map[string]struct{}) - for _, key := range excluded { - excludedSet[key] = struct{}{} - } - - included := make([]string, 0, len(mapping.Pairs())) - for _, pair := range mapping.Pairs() { - key, ok := pair.Key.AsString() - if !ok { - return dyn.InvalidValue, fmt.Errorf("expected a string key, but found %s", pair.Key.Kind()) - } - - if _, ok := excludedSet[key]; !ok { - included = append(included, pair.Key.MustString()) - } - } - - return Select(value, included) -} diff --git a/libs/dyn/merge/select_test.go b/libs/dyn/merge/select_test.go deleted file mode 100644 index c621f226849..00000000000 --- a/libs/dyn/merge/select_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package merge - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestSelect(t *testing.T) { - locations := []diag.Location{{File: "foo.yml", Line: 1, Column: 1}} - included := []string{"foo"} - input := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V("bar"), - "baz": dyn.V("qux"), - }, - locations, - ) - expected := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V("bar"), - }, - locations, - ) - - actual, err := Select(input, included) - - assert.NoError(t, err) - assert.Equal(t, expected, actual) -} - -func TestAntiSelect(t *testing.T) { - locations := []diag.Location{{File: "foo.yml", Line: 1, Column: 1}} - excluded := []string{"foo"} - input := dyn.NewValue( - map[string]dyn.Value{ - "foo": dyn.V("bar"), - "baz": dyn.V("qux"), - }, - locations, - ) - expected := dyn.NewValue( - map[string]dyn.Value{ - "baz": dyn.V("qux"), - }, - locations, - ) - - actual, err := AntiSelect(input, excluded) - - assert.NoError(t, err) - assert.Equal(t, expected, actual) -} diff --git a/libs/dyn/path.go b/libs/dyn/path.go deleted file mode 100644 index 60579103ca8..00000000000 --- a/libs/dyn/path.go +++ /dev/null @@ -1,146 +0,0 @@ -package dyn - -import ( - "bytes" - "fmt" -) - -type pathComponent struct { - key string - index int -} - -func (c pathComponent) Key() string { - return c.key -} - -func (c pathComponent) Index() int { - return c.index -} - -func (c pathComponent) isKey() bool { - return c.key != "" -} - -func (c pathComponent) isIndex() bool { - return c.key == "" -} - -// Path represents a path to a value in a [Value] configuration tree. -type Path []pathComponent - -// EmptyPath is the empty path. -// It is defined for convenience and clarity. -var EmptyPath = Path{} - -// Key returns a path component for a key. -func Key(k string) pathComponent { - return pathComponent{key: k} -} - -// Index returns a path component for an index. -func Index(i int) pathComponent { - return pathComponent{index: i} -} - -// NewPath returns a new path from the given components. -// The individual components may be created with [Key] or [Index]. -func NewPath(cs ...pathComponent) Path { - return cs -} - -// Append appends the given components to the path. -// Mutations to the returned path do not affect the original path. -func (p Path) Append(cs ...pathComponent) Path { - out := make(Path, len(p)+len(cs)) - copy(out, p) - copy(out[len(p):], cs) - return out -} - -// Equal returns true if the paths are equal. -func (p Path) Equal(q Path) bool { - pl := len(p) - ql := len(q) - if pl != ql { - return false - } - for i := range pl { - if p[i] != q[i] { - return false - } - } - return true -} - -// HasPrefix returns true if the path has the specified prefix. -// The empty path is a prefix of all paths. -func (p Path) HasPrefix(q Path) bool { - pl := len(p) - ql := len(q) - if pl < ql { - return false - } - for i := range ql { - if p[i] != q[i] { - return false - } - } - return true -} - -// HasSuffix returns true if the path has the specified suffix. -// The empty path is a suffix of all paths. -func (p Path) HasSuffix(q Path) bool { - pl := len(p) - ql := len(q) - if pl < ql { - return false - } - for i := range ql { - if p[pl-ql+i] != q[i] { - return false - } - } - return true -} - -// CutPrefix returns the path with the specified prefix removed. -// If the path does not have the specified prefix, the original path is returned. -// The second return value is true if the prefix was removed. -// Logically equivalent to [strings.CutPrefix]. -func (p Path) CutPrefix(q Path) (Path, bool) { - if !p.HasPrefix(q) { - return p, false - } - return p[len(q):], true -} - -// CutSuffix returns the path with the specified suffix removed. -// If the path does not have the specified suffix, the original path is returned. -// The second return value is true if the suffix was removed. -// Logically equivalent to [strings.CutSuffix]. -func (p Path) CutSuffix(q Path) (Path, bool) { - if !p.HasSuffix(q) { - return p, false - } - return p[:len(p)-len(q)], true -} - -// String returns a string representation of the path. -func (p Path) String() string { - var buf bytes.Buffer - - for i, c := range p { - if i > 0 && c.key != "" { - buf.WriteRune('.') - } - if c.key != "" { - buf.WriteString(c.key) - } else { - fmt.Fprintf(&buf, "[%d]", c.index) - } - } - - return buf.String() -} diff --git a/libs/dyn/path_string.go b/libs/dyn/path_string.go deleted file mode 100644 index 2f17f9279c2..00000000000 --- a/libs/dyn/path_string.go +++ /dev/null @@ -1,89 +0,0 @@ -package dyn - -import ( - "fmt" - "strconv" - "strings" -) - -// MustPathFromString is like NewPathFromString but panics on error. -func MustPathFromString(input string) Path { - p, err := NewPathFromString(input) - if err != nil { - panic(err) - } - return p -} - -// NewPathFromString parses a path from a string. -// -// The string must be a sequence of keys and indices separated by dots. -// Indices must be enclosed in square brackets. -// The string may include a leading dot. -// -// Examples: -// - foo.bar -// - foo[1].bar -// - foo.bar[1] -// - foo.bar[1][2] -// - . -func NewPathFromString(input string) (Path, error) { - var path Path - - p := input - - // Trim leading dot. - if p != "" && p[0] == '.' { - p = p[1:] - } - - for p != "" { - // Every component may have a leading dot. - if p[0] == '.' { - p = p[1:] - } - - if p == "" { - return nil, fmt.Errorf("invalid path: %s", input) - } - - if p[0] == '[' { - // Find next ] - i := strings.Index(p, "]") - if i < 0 { - return nil, fmt.Errorf("invalid path: %s", input) - } - - // Parse index - j, err := strconv.Atoi(p[1:i]) - if err != nil { - return nil, fmt.Errorf("invalid path: %s", input) - } - - // Append index - path = append(path, Index(j)) - p = p[i+1:] - - // The next character must be a . or [ - if p != "" && strings.IndexAny(p, ".[") != 0 { - return nil, fmt.Errorf("invalid path: %s", input) - } - } else { - // Find next . or [ - i := strings.IndexAny(p, ".[") - if i < 0 { - i = len(p) - } - - if i == 0 { - return nil, fmt.Errorf("invalid path: %s", input) - } - - // Append key - path = append(path, Key(p[:i])) - p = p[i:] - } - } - - return path, nil -} diff --git a/libs/dyn/path_string_test.go b/libs/dyn/path_string_test.go deleted file mode 100644 index d3bfbd7f493..00000000000 --- a/libs/dyn/path_string_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package dyn_test - -import ( - "errors" - "testing" - - . "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestNewPathFromString(t *testing.T) { - for _, tc := range []struct { - input string - output Path - err error - }{ - { - input: "", - output: NewPath(), - }, - { - input: ".", - output: NewPath(), - }, - { - input: "foo.bar", - output: NewPath(Key("foo"), Key("bar")), - }, - { - input: "[1]", - output: NewPath(Index(1)), - }, - { - input: "foo[1].bar", - output: NewPath(Key("foo"), Index(1), Key("bar")), - }, - { - input: "foo.bar[1]", - output: NewPath(Key("foo"), Key("bar"), Index(1)), - }, - { - input: "foo.bar[1][2]", - output: NewPath(Key("foo"), Key("bar"), Index(1), Index(2)), - }, - { - input: "foo.bar[1][2][3]", - output: NewPath(Key("foo"), Key("bar"), Index(1), Index(2), Index(3)), - }, - { - input: "foo[1234]", - output: NewPath(Key("foo"), Index(1234)), - }, - { - input: "foo[123", - err: errors.New("invalid path: foo[123"), - }, - { - input: "foo[123]]", - err: errors.New("invalid path: foo[123]]"), - }, - { - input: "foo[[123]", - err: errors.New("invalid path: foo[[123]"), - }, - { - input: "foo[[123]]", - err: errors.New("invalid path: foo[[123]]"), - }, - { - input: "foo[foo]", - err: errors.New("invalid path: foo[foo]"), - }, - { - input: "foo..bar", - err: errors.New("invalid path: foo..bar"), - }, - { - input: "foo.bar.", - err: errors.New("invalid path: foo.bar."), - }, - { - // Every component may have a leading dot. - input: ".foo.[1].bar", - output: NewPath(Key("foo"), Index(1), Key("bar")), - }, - { - // But after an index there must be a dot. - input: "foo[1]bar", - err: errors.New("invalid path: foo[1]bar"), - }, - { - // * is parsed as regular string in NewPathFromString - input: "foo.*", - output: NewPath(Key("foo"), Key("*")), - }, - { - // * is parsed as regular string in NewPathFromString - input: "foo.*.bar", - output: NewPath(Key("foo"), Key("*"), Key("bar")), - }, - { - // This is an invalid path (but would be valid for patterns) - input: "foo[*].bar", - err: errors.New("invalid path: foo[*].bar"), - }, - } { - p, err := NewPathFromString(tc.input) - if tc.err != nil { - assert.EqualError(t, err, tc.err.Error(), tc.input) - } else { - assert.NoError(t, err) - assert.Equal(t, tc.output, p) - } - } -} diff --git a/libs/dyn/path_test.go b/libs/dyn/path_test.go deleted file mode 100644 index 49114587175..00000000000 --- a/libs/dyn/path_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestPathAppend(t *testing.T) { - p := dyn.NewPath(dyn.Key("foo")) - - // Single arg. - p1 := p.Append(dyn.Key("bar")) - assert.True(t, p1.Equal(dyn.NewPath(dyn.Key("foo"), dyn.Key("bar")))) - - // Multiple args. - p2 := p.Append(dyn.Key("bar"), dyn.Index(1)) - assert.True(t, p2.Equal(dyn.NewPath(dyn.Key("foo"), dyn.Key("bar"), dyn.Index(1)))) -} - -func TestPathAppendAlwaysNew(t *testing.T) { - p := make(dyn.Path, 0, 2) - p = append(p, dyn.Key("foo")) - - // There is room for a second element in the slice. - p1 := p.Append(dyn.Index(1)) - p2 := p.Append(dyn.Index(2)) - assert.NotEqual(t, p1, p2) -} - -func TestPathEqualEmpty(t *testing.T) { - assert.True(t, dyn.EmptyPath.Equal(dyn.EmptyPath)) -} - -func TestPathEqual(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - p2 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2)) - assert.False(t, p1.Equal(p2), "expected %q to not equal %q", p1, p2) - - p3 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - assert.True(t, p1.Equal(p3), "expected %q to equal %q", p1, p3) - - p4 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar"), dyn.Index(2)) - assert.False(t, p1.Equal(p4), "expected %q to not equal %q", p1, p4) -} - -func TestPathHasPrefixEmpty(t *testing.T) { - empty := dyn.EmptyPath - nonEmpty := dyn.NewPath(dyn.Key("foo")) - assert.True(t, empty.HasPrefix(empty)) - assert.True(t, nonEmpty.HasPrefix(empty)) - assert.False(t, empty.HasPrefix(nonEmpty)) -} - -func TestPathHasPrefix(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - p2 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2)) - assert.False(t, p1.HasPrefix(p2), "expected %q to not have prefix %q", p1, p2) - - p3 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar")) - assert.False(t, p1.HasPrefix(p3), "expected %q to not have prefix %q", p1, p3) - - p4 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - assert.True(t, p1.HasPrefix(p4), "expected %q to have prefix %q", p1, p4) - - p5 := dyn.NewPath(dyn.Key("foo")) - assert.True(t, p1.HasPrefix(p5), "expected %q to have prefix %q", p1, p5) -} - -func TestPathHasSuffixEmpty(t *testing.T) { - empty := dyn.EmptyPath - nonEmpty := dyn.NewPath(dyn.Key("foo")) - assert.True(t, empty.HasSuffix(empty)) - assert.True(t, nonEmpty.HasSuffix(empty)) - assert.False(t, empty.HasSuffix(nonEmpty)) -} - -func TestPathHasSuffix(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - p2 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2)) - assert.False(t, p1.HasSuffix(p2), "expected %q to not have suffix %q", p1, p2) - - p3 := dyn.NewPath(dyn.Index(1)) - assert.True(t, p1.HasSuffix(p3), "expected %q to have suffix %q", p1, p3) - - p4 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - assert.True(t, p1.HasSuffix(p4), "expected %q to have suffix %q", p1, p4) - - p5 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2), dyn.Key("baz")) - assert.False(t, p1.HasSuffix(p5), "expected %q to not have suffix %q", p1, p5) -} - -func TestPathCutPrefix(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar")) - prefix := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - - // Cut a valid prefix. - rest, ok := p1.CutPrefix(prefix) - assert.True(t, ok, "expected %q to have prefix %q", p1, prefix) - assert.True(t, rest.Equal(dyn.NewPath(dyn.Key("bar"))), "expected rest to be %q, got %q", dyn.NewPath(dyn.Key("bar")), rest) - - // Try to cut an invalid prefix. - invalidPrefix := dyn.NewPath(dyn.Key("bar")) - rest, ok = p1.CutPrefix(invalidPrefix) - assert.False(t, ok, "expected %q to not have prefix %q", p1, invalidPrefix) - assert.True(t, rest.Equal(p1), "expected rest to be %q, got %q", p1, rest) - - // Cut an empty prefix. - emptyPrefix := dyn.EmptyPath - rest, ok = p1.CutPrefix(emptyPrefix) - assert.True(t, ok, "expected %q to have prefix %q", p1, emptyPrefix) - assert.True(t, rest.Equal(p1), "expected rest to be %q, got %q", p1, rest) - - // Cut a prefix equal to the path. - rest, ok = p1.CutPrefix(p1) - assert.True(t, ok, "expected %q to have prefix %q", p1, p1) - assert.True(t, rest.Equal(dyn.EmptyPath), "expected rest to be %q, got %q", dyn.EmptyPath, rest) -} - -func TestPathCutSuffix(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar")) - suffix := dyn.NewPath(dyn.Index(1), dyn.Key("bar")) - - // Cut a valid suffix. - rest, ok := p1.CutSuffix(suffix) - assert.True(t, ok, "expected %q to have suffix %q", p1, suffix) - assert.True(t, rest.Equal(dyn.NewPath(dyn.Key("foo"))), "expected rest to be %q, got %q", dyn.NewPath(dyn.Key("foo")), rest) - - // Try to cut an invalid suffix. - invalidSuffix := dyn.NewPath(dyn.Key("foo")) - rest, ok = p1.CutSuffix(invalidSuffix) - assert.False(t, ok, "expected %q to not have suffix %q", p1, invalidSuffix) - assert.True(t, rest.Equal(p1), "expected rest to be %q, got %q", p1, rest) - - // Cut an empty suffix. - emptySuffix := dyn.EmptyPath - rest, ok = p1.CutSuffix(emptySuffix) - assert.True(t, ok, "expected %q to have suffix %q", p1, emptySuffix) - assert.True(t, rest.Equal(p1), "expected rest to be %q, got %q", p1, rest) - - // Cut a suffix equal to the path. - rest, ok = p1.CutSuffix(p1) - assert.True(t, ok, "expected %q to have suffix %q", p1, p1) - assert.True(t, rest.Equal(dyn.EmptyPath), "expected rest to be %q, got %q", dyn.EmptyPath, rest) -} - -func TestPathString(t *testing.T) { - p1 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1)) - assert.Equal(t, "foo[1]", p1.String()) - - p2 := dyn.NewPath(dyn.Key("bar"), dyn.Index(2), dyn.Key("baz")) - assert.Equal(t, "bar[2].baz", p2.String()) - - p3 := dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar"), dyn.Index(2), dyn.Key("baz")) - assert.Equal(t, "foo[1].bar[2].baz", p3.String()) -} diff --git a/libs/dyn/pattern.go b/libs/dyn/pattern.go deleted file mode 100644 index cc53a2b0480..00000000000 --- a/libs/dyn/pattern.go +++ /dev/null @@ -1,191 +0,0 @@ -package dyn - -import ( - "errors" - "fmt" - "slices" - "strings" -) - -// Pattern represents a matcher for paths in a [Value] configuration tree. -// It is used by [MapByPattern] to apply a function to the values whose paths match the pattern. -// Every [Path] is a valid [Pattern] that matches a single unique path. -// The reverse is not true; not every [Pattern] is a valid [Path], as patterns may contain wildcards. -type Pattern []patternComponent - -func (p Pattern) String() string { - buf := strings.Builder{} - first := true - - for _, c := range p { - switch c := c.(type) { - case anyKeyComponent: - if !first { - buf.WriteString(".") - } - buf.WriteString("*") - case anyIndexComponent: - buf.WriteString("[*]") - case pathComponent: - if c.isKey() { - if !first { - buf.WriteString(".") - } - buf.WriteString(c.Key()) - } else { - fmt.Fprintf(&buf, "[%d]", c.Index()) - } - default: - buf.WriteString("???") - } - - first = false - } - return buf.String() -} - -// A pattern component can visit a [Value] and recursively call into [visit] for matching elements. -// Fixed components can match a single key or index, while wildcards can match any key or index. -type patternComponent interface { - visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) -} - -// NewPattern returns a new pattern from the given components. -// The individual components may be created with [Key], [Index], or [Any]. -func NewPattern(cs ...patternComponent) Pattern { - return cs -} - -// NewPatternFromPath returns a new pattern from the given path. -func NewPatternFromPath(p Path) Pattern { - cs := make(Pattern, len(p)) - for i, c := range p { - cs[i] = c - } - return cs -} - -// Split . into and -// The last component must be dyn.Key() and there must be at least two components. -func (p Pattern) SplitKey() (Pattern, string) { - if len(p) <= 1 { - return nil, "" - } - parent := p[:len(p)-1] - leaf := p[len(p)-1] - pc, ok := leaf.(pathComponent) - if !ok { - return nil, "" - } - key := pc.Key() - if key == "" { - return nil, "" - } - return parent, key -} - -// Append appends the given components to the pattern. -func (p Pattern) Append(cs ...patternComponent) Pattern { - out := make(Pattern, len(p)+len(cs)) - copy(out, p) - copy(out[len(p):], cs) - return out -} - -type anyKeyComponent struct{} - -// AnyKey returns a pattern component that matches any key. -func AnyKey() patternComponent { - return anyKeyComponent{} -} - -type expectedMapError struct { - p Path - v Value -} - -func (e expectedMapError) Error() string { - return fmt.Sprintf("expected a map at %q, found %s", e.p, e.v.Kind()) -} - -type expectedSequenceError struct { - p Path - v Value -} - -func (e expectedSequenceError) Error() string { - return fmt.Sprintf("expected a sequence at %q, found %s", e.p, e.v.Kind()) -} - -// isNoMatchError reports whether err means the pattern suffix didn't match the -// visited value. Wildcard components skip such elements (e.g. a job with an -// empty "tasks:" block) instead of failing the visit for all valid siblings. -func isNoMatchError(err error) bool { - if IsNoSuchKeyError(err) || IsIndexOutOfBoundsError(err) || IsCannotTraverseNilError(err) { - return true - } - if _, ok := errors.AsType[expectedMapError](err); ok { - return true - } - _, ok := errors.AsType[expectedSequenceError](err) - return ok -} - -// This function implements the patternComponent interface. -func (c anyKeyComponent) visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { - m, ok := v.AsMap() - if !ok { - return InvalidValue, expectedMapError{p: prefix, v: v} - } - - m = m.Clone() - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - - var err error - nv, err := visit(pv, append(prefix, Key(pk.MustString())), suffix, opts) - if err != nil { - // Leave the value intact if the suffix pattern didn't match any value. - if isNoMatchError(err) { - continue - } - return InvalidValue, err - } - - m.SetLoc(pk.MustString(), pk.Locations(), nv) - } - - return NewValue(m, v.Locations()), nil -} - -type anyIndexComponent struct{} - -// AnyIndex returns a pattern component that matches any index. -func AnyIndex() patternComponent { - return anyIndexComponent{} -} - -// This function implements the patternComponent interface. -func (c anyIndexComponent) visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { - s, ok := v.AsSequence() - if !ok { - return InvalidValue, expectedSequenceError{p: prefix, v: v} - } - - s = slices.Clone(s) - for i, value := range s { - var err error - nv, err := visit(value, append(prefix, Index(i)), suffix, opts) - if err != nil { - // Leave the value intact if the suffix pattern didn't match any value. - if isNoMatchError(err) { - continue - } - return InvalidValue, err - } - s[i] = nv - } - - return NewValue(s, v.Locations()), nil -} diff --git a/libs/dyn/pattern_string.go b/libs/dyn/pattern_string.go deleted file mode 100644 index 6d770faeb92..00000000000 --- a/libs/dyn/pattern_string.go +++ /dev/null @@ -1,102 +0,0 @@ -package dyn - -import ( - "fmt" - "strconv" - "strings" -) - -// MustPatternFromString is like NewPatternFromString but panics on error. -func MustPatternFromString(input string) Pattern { - p, err := NewPatternFromString(input) - if err != nil { - panic(err) - } - return p -} - -// NewPatternFromString parses a pattern from a string. -// -// The string must be a sequence of keys and indices separated by dots. -// Indices must be enclosed in square brackets. -// The string may include a leading dot. -// The wildcard character '*' can be used to match any key or index. -// -// Examples: -// - foo.bar -// - foo[1].bar -// - foo.*.bar -// - foo[*].bar -// - . -func NewPatternFromString(input string) (Pattern, error) { - var pattern Pattern - - p := input - - // Trim leading dot. - if p != "" && p[0] == '.' { - p = p[1:] - } - - for p != "" { - // Every component may have a leading dot. - if p[0] == '.' { - p = p[1:] - } - - if p == "" { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - if p[0] == '[' { - // Find next ] - i := strings.Index(p, "]") - if i < 0 { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - // Check for wildcard - if p[1:i] == "*" { - pattern = append(pattern, AnyIndex()) - } else { - // Parse index - j, err := strconv.Atoi(p[1:i]) - if err != nil { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - // Append index - pattern = append(pattern, Index(j)) - } - - p = p[i+1:] - - // The next character must be a . or [ - if p != "" && strings.IndexAny(p, ".[") != 0 { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - } else { - // Find next . or [ - i := strings.IndexAny(p, ".[") - if i < 0 { - i = len(p) - } - - if i == 0 { - return nil, fmt.Errorf("invalid pattern: %s", input) - } - - // Check for wildcard - if p[:i] == "*" { - pattern = append(pattern, AnyKey()) - } else { - // Append key - pattern = append(pattern, Key(p[:i])) - } - - p = p[i:] - } - } - - return pattern, nil -} diff --git a/libs/dyn/pattern_string_test.go b/libs/dyn/pattern_string_test.go deleted file mode 100644 index 85127c12ed1..00000000000 --- a/libs/dyn/pattern_string_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package dyn_test - -import ( - "errors" - "fmt" - "testing" - - . "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestNewPatternFromString(t *testing.T) { - for ind, tc := range []struct { - input string - output Pattern - err error - }{ - { - input: "", - output: NewPattern(), - }, - { - input: ".", - output: NewPattern(), - }, - { - input: "foo.bar", - output: NewPattern(Key("foo"), Key("bar")), - }, - { - input: "[1]", - output: NewPattern(Index(1)), - }, - { - input: "foo[1].bar", - output: NewPattern(Key("foo"), Index(1), Key("bar")), - }, - { - input: "foo.bar[1]", - output: NewPattern(Key("foo"), Key("bar"), Index(1)), - }, - { - input: "foo.bar[1][2]", - output: NewPattern(Key("foo"), Key("bar"), Index(1), Index(2)), - }, - { - input: "foo.bar[1][2][3]", - output: NewPattern(Key("foo"), Key("bar"), Index(1), Index(2), Index(3)), - }, - { - input: "foo[1234]", - output: NewPattern(Key("foo"), Index(1234)), - }, - { - input: "foo[123", - err: errors.New("invalid pattern: foo[123"), - }, - { - input: "foo[123]]", - err: errors.New("invalid pattern: foo[123]]"), - }, - { - input: "foo[[123]", - err: errors.New("invalid pattern: foo[[123]"), - }, - { - input: "foo[[123]]", - err: errors.New("invalid pattern: foo[[123]]"), - }, - { - input: "foo[foo]", - err: errors.New("invalid pattern: foo[foo]"), - }, - { - input: "foo..bar", - err: errors.New("invalid pattern: foo..bar"), - }, - { - input: "foo.bar.", - err: errors.New("invalid pattern: foo.bar."), - }, - { - // Every component may have a leading dot. - input: ".foo.[1].bar", - output: NewPattern(Key("foo"), Index(1), Key("bar")), - }, - { - // But after an index there must be a dot. - input: "foo[1]bar", - err: errors.New("invalid pattern: foo[1]bar"), - }, - // Wildcard tests - { - input: "foo.*", - output: NewPattern(Key("foo"), AnyKey()), - }, - { - input: "foo.*.bar", - output: NewPattern(Key("foo"), AnyKey(), Key("bar")), - }, - { - input: "foo[*]", - output: NewPattern(Key("foo"), AnyIndex()), - }, - { - input: "foo[*].bar", - output: NewPattern(Key("foo"), AnyIndex(), Key("bar")), - }, - { - input: "*[1]", - output: NewPattern(AnyKey(), Index(1)), - }, - { - input: "*.*", - output: NewPattern(AnyKey(), AnyKey()), - }, - { - input: "*[*]", - output: NewPattern(AnyKey(), AnyIndex()), - }, - } { - t.Run(fmt.Sprintf("%d %s", ind, tc.input), func(t *testing.T) { - p, err := NewPatternFromString(tc.input) - if tc.err != nil { - assert.EqualError(t, err, tc.err.Error(), tc.input) - } else { - assert.NoError(t, err) - assert.Equal(t, tc.output, p) - } - }) - } -} - -func TestMustPatternFromString(t *testing.T) { - // Test valid pattern - p := MustPatternFromString("foo.bar") - assert.Equal(t, NewPattern(Key("foo"), Key("bar")), p) - - // Test with wildcards - p = MustPatternFromString("foo.*.bar[*]") - assert.Equal(t, NewPattern(Key("foo"), AnyKey(), Key("bar"), AnyIndex()), p) - - // Test that invalid pattern panics - assert.Panics(t, func() { - MustPatternFromString("foo[") - }) -} diff --git a/libs/dyn/pattern_test.go b/libs/dyn/pattern_test.go deleted file mode 100644 index be78b6df61a..00000000000 --- a/libs/dyn/pattern_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package dyn_test - -import ( - "fmt" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestPatternString(t *testing.T) { - patterns := []string{ - "foo.bar", - "foo.bar.baz", - "foo[1]", - "foo[*]", - "*[1].bar", - "foo.*.bar", - "foo[*].bar", - "", - "foo", - "[1]", - "[*]", - "*", - } - - for _, p := range patterns { - pp := dyn.MustPatternFromString(p) - assert.Equal(t, p, pp.String()) - } -} - -func TestNewPattern(t *testing.T) { - pat := dyn.NewPattern( - dyn.Key("foo"), - dyn.Index(1), - ) - - assert.Len(t, pat, 2) -} - -func TestNewPatternFromPath(t *testing.T) { - path := dyn.NewPath( - dyn.Key("foo"), - dyn.Index(1), - ) - - pat1 := dyn.NewPattern(dyn.Key("foo"), dyn.Index(1)) - pat2 := dyn.NewPatternFromPath(path) - assert.Equal(t, pat1, pat2) -} - -func TestPatternAppend(t *testing.T) { - p := dyn.NewPattern(dyn.Key("foo")) - - // Single arg. - p1 := p.Append(dyn.Key("bar")) - assert.Equal(t, dyn.NewPattern(dyn.Key("foo"), dyn.Key("bar")), p1) - - // Multiple args. - p2 := p.Append(dyn.Key("bar"), dyn.Index(1)) - assert.Equal(t, dyn.NewPattern(dyn.Key("foo"), dyn.Key("bar"), dyn.Index(1)), p2) -} - -func TestPatternAppendAlwaysNew(t *testing.T) { - p := make(dyn.Pattern, 0, 2) - p = append(p, dyn.Key("foo")) - - // There is room for a second element in the slice. - p1 := p.Append(dyn.Index(1)) - p2 := p.Append(dyn.Index(2)) - assert.NotEqual(t, p1, p2) -} - -func TestPatternSplitKey(t *testing.T) { - p := dyn.NewPattern( - dyn.Key("foo"), - dyn.Key("bar"), - ) - - pat, key := p.SplitKey() - assert.Equal(t, "bar", key) - assert.Equal(t, dyn.NewPattern(dyn.Key("foo")), pat) -} - -func TestPatternSplitKeyError(t *testing.T) { - patterns := []dyn.Pattern{ - dyn.NewPattern( - dyn.Key("foo"), - dyn.AnyKey(), - ), - dyn.NewPattern( - dyn.Key("foo"), - dyn.AnyIndex(), - ), - dyn.NewPattern( - dyn.Key("foo"), - dyn.Index(1), - ), - dyn.NewPattern(), - } - - for ind, p := range patterns { - t.Run(fmt.Sprintf("%d %#v", ind, p), func(t *testing.T) { - pat, key := p.SplitKey() - assert.Empty(t, key) - assert.Empty(t, pat) - }) - } -} diff --git a/libs/dyn/structpath.go b/libs/dyn/structpath.go deleted file mode 100644 index 1cb740ee0d7..00000000000 --- a/libs/dyn/structpath.go +++ /dev/null @@ -1,41 +0,0 @@ -package dyn - -import "github.com/databricks/cli/libs/structs/structpath" - -// ToStructPath converts a [Path] to a [structpath.PathNode]. -func ToStructPath(p Path) *structpath.PathNode { - var result *structpath.PathNode - for _, c := range p { - if c.isKey() { - result = structpath.NewStringKey(result, c.key) - } else { - result = structpath.NewIndex(result, c.index) - } - } - return result -} - -// ToStructPaths converts each of paths with [ToStructPath], e.g. for diag.Diagnostic.Paths. -func ToStructPaths(paths ...Path) []*structpath.PathNode { - out := make([]*structpath.PathNode, len(paths)) - for i, p := range paths { - out[i] = ToStructPath(p) - } - return out -} - -// FromStructPath converts a [structpath.PathNode] of keys and indices to a [Path]. -// It reports false for nodes a [Path] cannot represent (key-value selectors, wildcards). -func FromStructPath(p *structpath.PathNode) (Path, bool) { - var out Path - for _, n := range p.AsSlice() { - if k, ok := n.StringKey(); ok { - out = append(out, Key(k)) - } else if i, ok := n.Index(); ok { - out = append(out, Index(i)) - } else { - return nil, false - } - } - return out, true -} diff --git a/libs/dyn/structpath_test.go b/libs/dyn/structpath_test.go deleted file mode 100644 index 6eed07a31fa..00000000000 --- a/libs/dyn/structpath_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/structs/structpath" - "github.com/stretchr/testify/assert" -) - -func TestToStructPath(t *testing.T) { - assert.Nil(t, dyn.ToStructPath(nil)) - - p := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("a_job"), dyn.Key("tasks"), dyn.Index(1), dyn.Key("task_key")) - assert.Equal(t, "resources.jobs.a_job.tasks[1].task_key", dyn.ToStructPath(p).String()) - - p = dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("${var.env}_job")) - assert.Equal(t, "resources.jobs['${var.env}_job']", dyn.ToStructPath(p).String()) -} - -func TestToStructPaths(t *testing.T) { - paths := dyn.ToStructPaths( - dyn.MustPathFromString("a.b"), - dyn.NewPath(dyn.Key("c"), dyn.Index(1)), - ) - assert.Len(t, paths, 2) - assert.Equal(t, "a.b", paths[0].String()) - assert.Equal(t, "c[1]", paths[1].String()) - assert.Empty(t, dyn.ToStructPaths()) -} - -func TestFromStructPath(t *testing.T) { - p, ok := dyn.FromStructPath(structpath.NewPath(nil, "resources", "jobs", "a.b", "tasks", 2)) - assert.True(t, ok) - assert.Equal(t, dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"), dyn.Key("a.b"), dyn.Key("tasks"), dyn.Index(2)), p) - - _, ok = dyn.FromStructPath(structpath.MustParsePath("tasks[task_key='x']")) - assert.False(t, ok) -} diff --git a/libs/dyn/suggest.go b/libs/dyn/suggest.go deleted file mode 100644 index ef8008aae6f..00000000000 --- a/libs/dyn/suggest.go +++ /dev/null @@ -1,108 +0,0 @@ -package dyn - -import ( - "fmt" - "slices" - "strings" -) - -const maxSuggestionDistance = 2 - -// levenshteinDistance computes the edit distance between two strings. -func levenshteinDistance(a, b string) int { - if len(a) == 0 { - return len(b) - } - if len(b) == 0 { - return len(a) - } - - // Use a single row for the DP table. - prev := make([]int, len(b)+1) - for j := range len(b) + 1 { - prev[j] = j - } - - for i := range len(a) { - curr := make([]int, len(b)+1) - curr[0] = i + 1 - for j := range len(b) { - cost := 1 - if a[i] == b[j] { - cost = 0 - } - curr[j+1] = min( - curr[j]+1, // insertion - prev[j+1]+1, // deletion - prev[j]+cost, // substitution - ) - } - prev = curr - } - - return prev[len(b)] -} - -// suggestKeys returns the keys in m whose edit distance from name is at most -// maxSuggestionDistance, ordered by increasing distance. It is used to build -// "did you mean" hints for a key that was not found in the map. -func suggestKeys(m Mapping, name string) []string { - type candidate struct { - key string - dist int - } - - var candidates []candidate - for _, kv := range m.Keys() { - key := kv.MustString() - d := levenshteinDistance(name, key) - if d <= maxSuggestionDistance { - candidates = append(candidates, candidate{key, d}) - } - } - - slices.SortStableFunc(candidates, func(a, b candidate) int { - return a.dist - b.dist - }) - - suggestions := make([]string, len(candidates)) - for i, c := range candidates { - suggestions[i] = c.key - } - return suggestions -} - -// didYouMean formats a suggestion clause like `, did you mean "x"?` (or, for -// multiple candidates, `, did you mean one of: "x", "y"?`). It returns an empty -// string when there are no suggestions. -func didYouMean(suggestions []string) string { - switch len(suggestions) { - case 0: - return "" - case 1: - return fmt.Sprintf(", did you mean %q?", suggestions[0]) - default: - quoted := make([]string, len(suggestions)) - for i, s := range suggestions { - quoted[i] = fmt.Sprintf("%q", s) - } - return fmt.Sprintf(", did you mean one of: %s?", strings.Join(quoted, ", ")) - } -} - -// replaceKey returns reference with the component matching failedKey swapped for -// replacement, or just replacement if reference can't be parsed or has no match. -func replaceKey(reference, failedKey, replacement string) string { - p, err := NewPathFromString(reference) - if err != nil { - return replacement - } - for i, c := range p { - if c.Key() == failedKey { - out := p.Append() - out[i] = Key(replacement) - return out.String() - } - } - return replacement -} diff --git a/libs/dyn/suggest_test.go b/libs/dyn/suggest_test.go deleted file mode 100644 index b0aa4a87168..00000000000 --- a/libs/dyn/suggest_test.go +++ /dev/null @@ -1,164 +0,0 @@ -package dyn - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestLevenshteinDistance(t *testing.T) { - tests := []struct { - a string - b string - want int - }{ - {"", "", 0}, - {"", "abc", 3}, - {"abc", "", 3}, - {"abc", "abc", 0}, - {"abc", "abd", 1}, - {"kitten", "sitting", 3}, - {"host", "hosts", 1}, - } - for _, tt := range tests { - assert.Equal(t, tt.want, levenshteinDistance(tt.a, tt.b), "levenshteinDistance(%q, %q)", tt.a, tt.b) - } -} - -func newSuggestMapping(keys ...string) Mapping { - var m Mapping - for _, k := range keys { - m.SetLoc(k, nil, V(k)) - } - return m -} - -func TestSuggestKeys(t *testing.T) { - tests := []struct { - name string - keys []string - typo string - want []string - }{ - { - // Keys within distance 2 are returned ordered by increasing - // distance; ties keep the map's insertion order. - name: "ordered by distance", - keys: []string{"host", "hosts", "token", "auth_type"}, - typo: "host", - want: []string{"host", "hosts"}, - }, - { - name: "no key close enough", - keys: []string{"host", "hosts", "token", "auth_type"}, - typo: "completely_different", - want: []string{}, - }, - { - // Distance-2 substitutions and insertions are both included. - name: "distance two included", - keys: []string{"profile", "prfile", "prof"}, - typo: "prfil", - want: []string{"prfile", "profile"}, - }, - { - name: "empty map", - keys: nil, - typo: "anything", - want: []string{}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, suggestKeys(newSuggestMapping(tt.keys...), tt.typo)) - }) - } -} - -func TestDidYouMean(t *testing.T) { - tests := []struct { - name string - suggestions []string - want string - }{ - {"nil", nil, ""}, - {"empty", []string{}, ""}, - {"single", []string{"host"}, `, did you mean "host"?`}, - {"multiple", []string{"host", "hosts"}, `, did you mean one of: "host", "hosts"?`}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, didYouMean(tt.suggestions)) - }) - } -} - -func TestSuggestedReferences(t *testing.T) { - tests := []struct { - name string - err error - reference string - want []string - }{ - { - name: "single suggestion", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("hst")), suggestions: []string{"host"}}, - reference: "var.hst", - want: []string{"var.host"}, - }, - { - name: "multiple suggestions", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("hst")), suggestions: []string{"host", "hosts"}}, - reference: "var.hst", - want: []string{"var.host", "var.hosts"}, - }, - { - name: "nested outer-key typo keeps suffix", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("clustr")), suggestions: []string{"cluster"}}, - reference: "var.clustr.spark_version", - want: []string{"var.cluster.spark_version"}, - }, - { - name: "deep leaf typo keeps prefix", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("cluster"), Key("value"), Key("spark_versio")), suggestions: []string{"spark_version"}}, - reference: "var.cluster.spark_versio", - want: []string{"var.cluster.spark_version"}, - }, - { - name: "index component preserved", - err: noSuchKeyError{p: NewPath(Key("variables"), Key("librariez")), suggestions: []string{"libraries"}}, - reference: "var.librariez[0].jar", - want: []string{"var.libraries[0].jar"}, - }, - { - name: "non-var prefix", - err: noSuchKeyError{p: NewPath(Key("workspace"), Key("stot_path")), suggestions: []string{"root_path", "state_path"}}, - reference: "workspace.stot_path", - want: []string{"workspace.root_path", "workspace.state_path"}, - }, - { - name: "no suggestions", - err: noSuchKeyError{p: NewPath(Key("xyz"))}, - reference: "var.xyz", - want: nil, - }, - { - name: "other error type", - err: errors.New("some other error"), - reference: "var.xyz", - want: nil, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, SuggestedReferences(tt.err, tt.reference)) - }) - } -} - -func TestReplaceKeyFallsBackWhenKeyAbsent(t *testing.T) { - // The failed key is not present in the reference, so only the replacement - // itself is returned rather than a spliced reference. - assert.Equal(t, "host", replaceKey("var.foo", "missing", "host")) -} diff --git a/libs/dyn/time.go b/libs/dyn/time.go deleted file mode 100644 index c7f043f06ca..00000000000 --- a/libs/dyn/time.go +++ /dev/null @@ -1,66 +0,0 @@ -package dyn - -import ( - "fmt" - "time" -) - -// Time represents a time-like primitive value. -// -// It represents a timestamp and includes the original string value -// that was parsed to create the timestamp. This makes it possible -// to coalesce a value that YAML interprets as a timestamp back into -// a string without losing information. -type Time struct { - t time.Time - s string -} - -// NewTime creates a new Time from the given string. -func NewTime(str string) (Time, error) { - // Try a couple of layouts - for _, layout := range []string{ - "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. - "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". - "2006-1-2 15:4:5.999999999", // space separated with no time zone - "2006-1-2", // date only - } { - t, terr := time.Parse(layout, str) - if terr == nil { - return Time{t: t, s: str}, nil - } - } - - return Time{}, fmt.Errorf("invalid time value: %q", str) -} - -// MustTime creates a new Time from the given string. -// It panics if the string cannot be parsed. -func MustTime(str string) Time { - t, err := NewTime(str) - if err != nil { - panic(err) - } - return t -} - -// FromTime creates a new Time from the given time.Time. -// It uses the RFC3339Nano format for its string representation. -// This guarantees that it can roundtrip into a string without losing information. -func FromTime(t time.Time) Time { - return Time{t: t, s: t.Format(time.RFC3339Nano)} -} - -// Time returns the time.Time value. -func (t Time) Time() time.Time { - return t.t -} - -// String returns the original string value that was parsed to create the timestamp. -func (t Time) String() string { - return t.s -} - -func (t Time) IsZero() bool { - return t.s == "" && t.t.IsZero() -} diff --git a/libs/dyn/time_test.go b/libs/dyn/time_test.go deleted file mode 100644 index 61cc4fef93d..00000000000 --- a/libs/dyn/time_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package dyn_test - -import ( - "testing" - "time" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestTimeValid(t *testing.T) { - for _, tc := range []string{ - "2024-08-29", - "2024-01-15T12:34:56.789012345Z", - } { - tm, err := dyn.NewTime(tc) - if assert.NoError(t, err) { - assert.NotEqual(t, time.Time{}, tm.Time()) - assert.Equal(t, tc, tm.String()) - } - } -} - -func TestTimeInvalid(t *testing.T) { - tm, err := dyn.NewTime("invalid") - assert.Error(t, err) - assert.Equal(t, dyn.Time{}, tm) -} - -func TestTimeFromTime(t *testing.T) { - tref := time.Now() - t1 := dyn.FromTime(tref) - - // Verify that the underlying value is the same. - assert.Equal(t, tref, t1.Time()) - - // Verify that the string representation can be used to construct the same. - t2, err := dyn.NewTime(t1.String()) - assert.NoError(t, err) - assert.True(t, t1.Time().Equal(t2.Time())) -} diff --git a/libs/dyn/value.go b/libs/dyn/value.go deleted file mode 100644 index 8cbd9b801ea..00000000000 --- a/libs/dyn/value.go +++ /dev/null @@ -1,245 +0,0 @@ -package dyn - -import ( - "fmt" - "reflect" - "slices" - - "github.com/databricks/cli/libs/diag" -) - -type Value struct { - v any - - k Kind - - // List of locations this value is defined at. The first location in the slice - // is the location returned by the `.Location()` method and is typically used - // for reporting errors and warnings associated with the value. - l []diag.Location - - // Whether or not this value is an anchor. - // If this node doesn't map to a type, we don't need to warn about it. - anchor bool -} - -// InvalidValue is equal to the zero-value of Value. -var InvalidValue = Value{ - k: KindInvalid, -} - -// NilValue is a convenient constant for a nil value. -var NilValue = Value{ - k: KindNil, -} - -// V constructs a new Value with the given value. -func V(v any) Value { - return NewValue(v, nil) -} - -// NewValue constructs a new Value with the given value and location. -func NewValue(v any, loc []diag.Location) Value { - switch vin := v.(type) { - case map[string]Value: - v = newMappingFromGoMap(vin) - } - - return Value{ - v: v, - k: kindOf(v), - - // create a copy of the locations, so that mutations to the original slice - // don't affect new value. - l: slices.Clone(loc), - } -} - -// WithLocations returns a new Value with its location set to the given value. -func (v Value) WithLocations(loc []diag.Location) Value { - return Value{ - v: v.v, - k: v.k, - - // create a copy of the locations, so that mutations to the original slice - // don't affect new value. - l: slices.Clone(loc), - } -} - -func (v Value) AppendLocationsFromValue(w Value) Value { - return Value{ - v: v.v, - k: v.k, - l: append(v.l, w.l...), - } -} - -func (v Value) Kind() Kind { - return v.k -} - -func (v Value) Value() any { - return v.v -} - -func (v Value) Locations() []diag.Location { - return v.l -} - -func (v Value) Location() diag.Location { - if len(v.l) == 0 { - return diag.Location{} - } - - return v.l[0] -} - -func (v Value) IsValid() bool { - return v.k != KindInvalid -} - -func (v Value) AsAny() any { - switch v.k { - case KindInvalid: - panic("invoked AsAny on invalid value") - case KindMap: - m := v.v.(Mapping) - out := make(map[string]any, m.Len()) - for _, pair := range m.pairs { - pk := pair.Key - pv := pair.Value - out[pk.MustString()] = pv.AsAny() - } - return out - case KindSequence: - vv := v.v.([]Value) - a := make([]any, len(vv)) - for i, v := range vv { - a[i] = v.AsAny() - } - return a - case KindNil: - return v.v - case KindString: - return v.v - case KindBool: - return v.v - case KindInt: - return v.v - case KindFloat: - return v.v - case KindTime: - t := v.v.(Time) - return t.Time() - default: - // Panic because we only want to deal with known types. - panic(fmt.Sprintf("invalid kind: %d", v.k)) - } -} - -func (v Value) IsZero() bool { - if v.v == nil { - return true - } - - switch x := v.v.(type) { - case Mapping: - return x.Len() == 0 - case []Value: - return len(x) == 0 - default: - return reflect.ValueOf(x).IsZero() - } -} - -func (v Value) Get(key string) Value { - m, ok := v.AsMap() - if !ok { - return InvalidValue - } - - vv, ok := m.GetByString(key) - if !ok { - return InvalidValue - } - - return vv -} - -func (v Value) Index(i int) Value { - s, ok := v.v.([]Value) - if !ok { - return InvalidValue - } - - if i < 0 || i >= len(s) { - return InvalidValue - } - - return s[i] -} - -func (v Value) MarkAnchor() Value { - return Value{ - v: v.v, - k: v.k, - l: v.l, - - anchor: true, - } -} - -func (v Value) IsAnchor() bool { - return v.anchor -} - -// eq is an internal only method that compares two values. -// It is used to determine if a value has changed during a visit. -// We need a custom implementation because maps and slices -// cannot be compared with the regular == operator. -func (v Value) eq(w Value) bool { - if v.k != w.k { - return false - } - if !slices.Equal(v.l, w.l) { - return false - } - - switch v.k { - case KindMap: - // Comparing &v.v to &w.v is always false (value receivers), so compare the Mapping's backing storage. - vm := v.v.(Mapping) - wm := w.v.(Mapping) - lv := vm.Len() - lw := wm.Len() - if lv == 0 && lw == 0 { - return true - } - if lv != lw { - return false - } - // Compare pointers to the underlying pairs slice. - // This is safe because we don't allow maps to be mutated. - return &vm.pairs[0] == &wm.pairs[0] - case KindSequence: - vs := v.v.([]Value) - ws := w.v.([]Value) - lv := len(vs) - lw := len(ws) - // If both slices are empty, they are equal. - if lv == 0 && lw == 0 { - return true - } - // If they have different lengths, they are not equal. - if lv != lw { - return false - } - // They are both non-empty and have the same length. - // Compare pointers to the underlying slice. - // This is safe because we don't allow slices to be mutated. - return &vs[0] == &ws[0] - default: - return v.v == w.v - } -} diff --git a/libs/dyn/value_eq_test.go b/libs/dyn/value_eq_test.go deleted file mode 100644 index 22fe2df0080..00000000000 --- a/libs/dyn/value_eq_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package dyn - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestValueEqMap(t *testing.T) { - loc := []diag.Location{{File: "file", Line: 1, Column: 2}} - v := NewValue(map[string]Value{"key": V("value")}, loc) - - tests := []struct { - name string - a, b Value - want bool - }{ - { - name: "same underlying mapping", - a: v, - b: v, - want: true, - }, - { - name: "cloned mapping with equal contents", - a: v, - b: Value{v: v.v.(Mapping).Clone(), k: KindMap, l: v.l}, - want: false, - }, - { - name: "different lengths", - a: v, - b: NewValue(map[string]Value{"key": V("value"), "other": V("value")}, loc), - want: false, - }, - { - name: "different locations", - a: v, - b: v.WithLocations([]diag.Location{{File: "other", Line: 1, Column: 2}}), - want: false, - }, - { - name: "empty mappings", - a: NewValue(map[string]Value{}, loc), - b: NewValue(map[string]Value{}, loc), - want: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.want, tc.a.eq(tc.b)) - }) - } -} - -func TestValueEqMapVisitDoesNotRebuildAncestors(t *testing.T) { - vin := V(map[string]Value{ - "a": V(map[string]Value{ - "b": V("value"), - }), - }) - - vout, err := Map(vin, "a.b", func(_ Path, v Value) (Value, error) { - return v, nil - }) - require.NoError(t, err) - - // The identity transform must return the original value without - // cloning the ancestor maps. - vm := vin.v.(Mapping) - wm := vout.v.(Mapping) - require.Equal(t, vm.Len(), wm.Len()) - assert.Same(t, &vm.pairs[0], &wm.pairs[0]) -} diff --git a/libs/dyn/value_test.go b/libs/dyn/value_test.go deleted file mode 100644 index 667e2cbdd59..00000000000 --- a/libs/dyn/value_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestInvalidValue(t *testing.T) { - // Assert that the zero value of [dyn.Value] is the invalid value. - var zero dyn.Value - assert.Equal(t, zero, dyn.InvalidValue) -} - -func TestValueIsAnchor(t *testing.T) { - var zero dyn.Value - assert.False(t, zero.IsAnchor()) - mark := zero.MarkAnchor() - assert.True(t, mark.IsAnchor()) -} - -func TestValueAsMap(t *testing.T) { - var zeroValue dyn.Value - _, ok := zeroValue.AsMap() - assert.False(t, ok) - - intValue := dyn.V(1) - _, ok = intValue.AsMap() - assert.False(t, ok) - - mapValue := dyn.NewValue( - map[string]dyn.Value{ - "key": dyn.NewValue( - "value", - []diag.Location{{File: "file", Line: 1, Column: 2}}), - }, - []diag.Location{{File: "file", Line: 1, Column: 2}}, - ) - - m, ok := mapValue.AsMap() - assert.True(t, ok) - assert.Equal(t, 1, m.Len()) -} - -func TestValueIsValid(t *testing.T) { - var zeroValue dyn.Value - assert.False(t, zeroValue.IsValid()) - intValue := dyn.V(1) - assert.True(t, intValue.IsValid()) -} - -func TestIsZero(t *testing.T) { - assert.True(t, dyn.V(0).IsZero(), "int") - assert.True(t, dyn.V(int(0)).IsZero(), "int") - assert.False(t, dyn.V(int(1)).IsZero(), "int") - assert.True(t, dyn.V(uint(0)).IsZero(), "uint") - - assert.True(t, dyn.V(int8(0)).IsZero(), "int8") - assert.True(t, dyn.V(uint8(0)).IsZero(), "uint8") - - assert.True(t, dyn.V(int16(0)).IsZero(), "int16") - assert.True(t, dyn.V(uint16(0)).IsZero(), "uint16") - - assert.True(t, dyn.V(int32(0)).IsZero(), "int32") - assert.False(t, dyn.V(int32(1)).IsZero(), "int32") - assert.True(t, dyn.V(uint32(0)).IsZero(), "uint32") - - assert.True(t, dyn.V(int64(0)).IsZero(), "int64") - assert.False(t, dyn.V(int64(-1)).IsZero(), "int64") - - assert.True(t, dyn.V(uint64(0)).IsZero(), "uint64") - assert.False(t, dyn.V(uint64(2)).IsZero(), "uint64") - - assert.True(t, dyn.V("").IsZero(), "string") - assert.False(t, dyn.V("x").IsZero(), "string") - - assert.True(t, dyn.V(false).IsZero(), "bool") - assert.False(t, dyn.V(true).IsZero(), "bool") - - assert.True(t, dyn.V(float32(0.0)).IsZero(), "float32") - assert.False(t, dyn.V(float32(0.01)).IsZero(), "float32") - - assert.True(t, dyn.V(float64(0.0)).IsZero(), "float64") - assert.False(t, dyn.V(float64(0.01)).IsZero(), "float64") - - assert.True(t, dyn.V(dyn.Time{}).IsZero(), "time") - assert.True(t, dyn.V(dyn.Mapping{}).IsZero(), "Mapping") - assert.True(t, dyn.V([]dyn.Value{}).IsZero(), "Sequence") - assert.False(t, dyn.V([]dyn.Value{dyn.V(0)}).IsZero(), "Sequence") -} diff --git a/libs/dyn/value_underlying.go b/libs/dyn/value_underlying.go deleted file mode 100644 index a33ecd38ed8..00000000000 --- a/libs/dyn/value_underlying.go +++ /dev/null @@ -1,138 +0,0 @@ -package dyn - -import ( - "fmt" -) - -// AsMap returns the underlying mapping if this value is a map, -// the zero value and false otherwise. -func (v Value) AsMap() (Mapping, bool) { - vv, ok := v.v.(Mapping) - return vv, ok -} - -// MustMap returns the underlying mapping if this value is a map, -// panics otherwise. -func (v Value) MustMap() Mapping { - vv, ok := v.AsMap() - if !ok || v.k != KindMap { - panic(fmt.Sprintf("expected kind %s, got %s", KindMap, v.k)) - } - return vv -} - -// AsSequence returns the underlying sequence if this value is a sequence, -// the zero value and false otherwise. -func (v Value) AsSequence() ([]Value, bool) { - vv, ok := v.v.([]Value) - return vv, ok -} - -// MustSequence returns the underlying sequence if this value is a sequence, -// panics otherwise. -func (v Value) MustSequence() []Value { - vv, ok := v.AsSequence() - if !ok || v.k != KindSequence { - panic(fmt.Sprintf("expected kind %s, got %s", KindSequence, v.k)) - } - return vv -} - -// AsString returns the underlying string if this value is a string, -// the zero value and false otherwise. -func (v Value) AsString() (string, bool) { - vv, ok := v.v.(string) - return vv, ok -} - -// MustString returns the underlying string if this value is a string, -// panics otherwise. -func (v Value) MustString() string { - vv, ok := v.AsString() - if !ok || v.k != KindString { - panic(fmt.Sprintf("expected kind %s, got %s", KindString, v.k)) - } - return vv -} - -// AsBool returns the underlying bool if this value is a bool, -// the zero value and false otherwise. -func (v Value) AsBool() (bool, bool) { - vv, ok := v.v.(bool) - return vv, ok -} - -// MustBool returns the underlying bool if this value is a bool, -// panics otherwise. -func (v Value) MustBool() bool { - vv, ok := v.AsBool() - if !ok || v.k != KindBool { - panic(fmt.Sprintf("expected kind %s, got %s", KindBool, v.k)) - } - return vv -} - -// AsInt returns the underlying int if this value is an int, -// the zero value and false otherwise. -func (v Value) AsInt() (int64, bool) { - switch vv := v.v.(type) { - case int: - return int64(vv), true - case int32: - return int64(vv), true - case int64: - return vv, true - default: - return 0, false - } -} - -// MustInt returns the underlying int if this value is an int, -// panics otherwise. -func (v Value) MustInt() int64 { - vv, ok := v.AsInt() - if !ok || v.k != KindInt { - panic(fmt.Sprintf("expected kind %s, got %s", KindInt, v.k)) - } - return vv -} - -// AsFloat returns the underlying float if this value is a float, -// the zero value and false otherwise. -func (v Value) AsFloat() (float64, bool) { - switch vv := v.v.(type) { - case float32: - return float64(vv), true - case float64: - return float64(vv), true - default: - return 0, false - } -} - -// MustFloat returns the underlying float if this value is a float, -// panics otherwise. -func (v Value) MustFloat() float64 { - vv, ok := v.AsFloat() - if !ok || v.k != KindFloat { - panic(fmt.Sprintf("expected kind %s, got %s", KindFloat, v.k)) - } - return vv -} - -// AsTime returns the underlying time if this value is a time, -// the zero value and false otherwise. -func (v Value) AsTime() (Time, bool) { - vv, ok := v.v.(Time) - return vv, ok -} - -// MustTime returns the underlying time if this value is a time, -// panics otherwise. -func (v Value) MustTime() Time { - vv, ok := v.AsTime() - if !ok || v.k != KindTime { - panic(fmt.Sprintf("expected kind %s, got %s", KindTime, v.k)) - } - return vv -} diff --git a/libs/dyn/value_underlying_test.go b/libs/dyn/value_underlying_test.go deleted file mode 100644 index 189a49a0ea3..00000000000 --- a/libs/dyn/value_underlying_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package dyn_test - -import ( - "testing" - "time" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestValueUnderlyingMap(t *testing.T) { - v := dyn.V( - map[string]dyn.Value{ - "key": dyn.NewValue("value", []diag.Location{{File: "file", Line: 1, Column: 2}}), - }, - ) - - vv1, ok := v.AsMap() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsMap() - assert.False(t, ok) - - vv2 := v.MustMap() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind map, got invalid", func() { - dyn.InvalidValue.MustMap() - }) -} - -func TestValueUnderlyingSequence(t *testing.T) { - v := dyn.V( - []dyn.Value{ - dyn.NewValue("value", []diag.Location{{File: "file", Line: 1, Column: 2}}), - }, - ) - - vv1, ok := v.AsSequence() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsSequence() - assert.False(t, ok) - - vv2 := v.MustSequence() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind sequence, got invalid", func() { - dyn.InvalidValue.MustSequence() - }) -} - -func TestValueUnderlyingString(t *testing.T) { - v := dyn.V("value") - - vv1, ok := v.AsString() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsString() - assert.False(t, ok) - - vv2 := v.MustString() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind string, got invalid", func() { - dyn.InvalidValue.MustString() - }) -} - -func TestValueUnderlyingBool(t *testing.T) { - v := dyn.V(true) - - vv1, ok := v.AsBool() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsBool() - assert.False(t, ok) - - vv2 := v.MustBool() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind bool, got invalid", func() { - dyn.InvalidValue.MustBool() - }) -} - -func TestValueUnderlyingInt(t *testing.T) { - v := dyn.V(int(1)) - - vv1, ok := v.AsInt() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsInt() - assert.False(t, ok) - - vv2 := v.MustInt() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind int, got invalid", func() { - dyn.InvalidValue.MustInt() - }) - - // Test int32 type specifically. - v = dyn.V(int32(1)) - vv1, ok = v.AsInt() - assert.True(t, ok) - assert.Equal(t, int64(1), vv1) - - // Test int64 type specifically. - v = dyn.V(int64(1)) - vv1, ok = v.AsInt() - assert.True(t, ok) - assert.Equal(t, int64(1), vv1) -} - -func TestValueUnderlyingFloat(t *testing.T) { - v := dyn.V(float32(1.0)) - - vv1, ok := v.AsFloat() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsFloat() - assert.False(t, ok) - - vv2 := v.MustFloat() - assert.Zero(t, vv1-vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind float, got invalid", func() { - dyn.InvalidValue.MustFloat() - }) - - // Test float64 type specifically. - v = dyn.V(float64(1.0)) - vv1, ok = v.AsFloat() - assert.True(t, ok) - assert.Zero(t, 1.0-vv1) -} - -func TestValueUnderlyingTime(t *testing.T) { - v := dyn.V(dyn.FromTime(time.Now())) - - vv1, ok := v.AsTime() - assert.True(t, ok) - - _, ok = dyn.InvalidValue.AsTime() - assert.False(t, ok) - - vv2 := v.MustTime() - assert.Equal(t, vv1, vv2) - - // Test panic. - assert.PanicsWithValue(t, "expected kind time, got invalid", func() { - dyn.InvalidValue.MustTime() - }) -} diff --git a/libs/dyn/visit.go b/libs/dyn/visit.go deleted file mode 100644 index 58223e792b9..00000000000 --- a/libs/dyn/visit.go +++ /dev/null @@ -1,210 +0,0 @@ -package dyn - -import ( - "errors" - "fmt" - "slices" -) - -// This error is returned if the path indicates that a map or sequence is expected, but the value is nil. -type cannotTraverseNilError struct { - p Path -} - -func (e cannotTraverseNilError) Error() string { - component := e.p[len(e.p)-1] - switch { - case component.isKey(): - return fmt.Sprintf("expected a map to index %q, found nil", e.p) - case component.isIndex(): - return fmt.Sprintf("expected a sequence to index %q, found nil", e.p) - default: - panic("invalid component") - } -} - -func IsCannotTraverseNilError(err error) bool { - _, ok := errors.AsType[cannotTraverseNilError](err) - return ok -} - -type noSuchKeyError struct { - p Path - suggestions []string -} - -func (e noSuchKeyError) Error() string { - return fmt.Sprintf("key not found at %q%s", e.p, didYouMean(e.suggestions)) -} - -func IsNoSuchKeyError(err error) bool { - _, ok := errors.AsType[noSuchKeyError](err) - return ok -} - -// SuggestedReferences returns drop-in replacement references for a noSuchKeyError -// (nil otherwise), rebuilt by swapping the failed segment of reference for each -// suggestion (e.g. "var.hst" -> ["var.host"]). -func SuggestedReferences(err error, reference string) []string { - e, ok := errors.AsType[noSuchKeyError](err) - if !ok || len(e.suggestions) == 0 { - return nil - } - // Last component of e.p is the failed key (same in original and rewritten space). - failedKey := e.p[len(e.p)-1].Key() - refs := make([]string, len(e.suggestions)) - for i, s := range e.suggestions { - refs[i] = replaceKey(reference, failedKey, s) - } - return refs -} - -type indexOutOfBoundsError struct { - p Path -} - -func (e indexOutOfBoundsError) Error() string { - return fmt.Sprintf("index out of bounds at %q", e.p) -} - -func IsIndexOutOfBoundsError(err error) bool { - _, ok := errors.AsType[indexOutOfBoundsError](err) - return ok -} - -type expectedMapToIndexError struct { - p Path - v Value -} - -func (e expectedMapToIndexError) Error() string { - return fmt.Sprintf("expected a map to index %q, found %s", e.p, e.v.Kind()) -} - -type expectedSequenceToIndexError struct { - p Path - v Value -} - -func (e expectedSequenceToIndexError) Error() string { - return fmt.Sprintf("expected a sequence to index %q, found %s", e.p, e.v.Kind()) -} - -type visitOptions struct { - // The function to apply to the value once found. - // - // If this function returns the same value as it receives as argument, - // the original visit function call returns the original value unmodified. - // - // If this function returns a new value, the original visit function call - // returns a value with all the intermediate values updated. - // - // If this function returns an error, the original visit function call - // returns this error and the value is left unmodified. - fn func(Path, Value) (Value, error) -} - -func visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { - if len(suffix) == 0 { - return opts.fn(slices.Clone(prefix), v) - } - - // Initialize prefix if it is empty. - // It is pre-allocated to its maximum size to avoid additional allocations. - if len(prefix) == 0 { - prefix = make(Path, 0, len(suffix)) - } - - component := suffix[0] - suffix = suffix[1:] - - // Visit the value with the current component. - return component.visit(v, prefix, suffix, opts) -} - -func (c pathComponent) visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { - path := append(prefix, c) - - switch { - case c.isKey(): - // Expect a map to be set if this is a key. - switch v.Kind() { - case KindMap: - // OK - case KindNil: - return InvalidValue, cannotTraverseNilError{path} - default: - return InvalidValue, expectedMapToIndexError{p: path, v: v} - } - - m := v.MustMap() - - // Lookup current value in the map. - ev, ok := m.GetByString(c.key) - if !ok { - return InvalidValue, noSuchKeyError{p: path, suggestions: suggestKeys(m, c.key)} - } - - // Recursively transform the value. - nv, err := visit(ev, path, suffix, opts) - if err != nil { - return InvalidValue, err - } - - // Return the original value if the value hasn't changed. - if nv.eq(ev) { - return v, nil - } - - // Return an updated map value. - m = m.Clone() - m.SetLoc(c.key, nil, nv) - return Value{ - v: m, - k: KindMap, - l: v.l, - }, nil - - case c.isIndex(): - // Expect a sequence to be set if this is an index. - switch v.Kind() { - case KindSequence: - // OK - case KindNil: - return InvalidValue, cannotTraverseNilError{path} - default: - return InvalidValue, expectedSequenceToIndexError{p: path, v: v} - } - - s := v.MustSequence() - - // Lookup current value in the sequence. - if c.index < 0 || c.index >= len(s) { - return InvalidValue, indexOutOfBoundsError{path} - } - - // Recursively transform the value. - ev := s[c.index] - nv, err := visit(ev, path, suffix, opts) - if err != nil { - return InvalidValue, err - } - - // Return the original value if the value hasn't changed. - if nv.eq(ev) { - return v, nil - } - - // Return an updated sequence value. - s = slices.Clone(s) - s[c.index] = nv - return Value{ - v: s, - k: KindSequence, - l: v.l, - }, nil - - default: - panic("invalid component") - } -} diff --git a/libs/dyn/visit_get.go b/libs/dyn/visit_get.go deleted file mode 100644 index 101c38aff64..00000000000 --- a/libs/dyn/visit_get.go +++ /dev/null @@ -1,25 +0,0 @@ -package dyn - -// Get returns the value inside the specified value at the specified path. -// It is identical to [GetByPath], except that it takes a string path instead of a [Path]. -func Get(v Value, path string) (Value, error) { - p, err := NewPathFromString(path) - if err != nil { - return InvalidValue, err - } - return GetByPath(v, p) -} - -// GetByPath returns the value inside the specified value at the specified path. -// If the path doesn't exist, it returns InvalidValue and an error. -func GetByPath(v Value, p Path) (Value, error) { - out := InvalidValue - _, err := visit(v, EmptyPath, NewPatternFromPath(p), visitOptions{ - fn: func(_ Path, ev Value) (Value, error) { - // Capture the value argument to return it. - out = ev - return ev, nil - }, - }) - return out, err -} diff --git a/libs/dyn/visit_get_test.go b/libs/dyn/visit_get_test.go deleted file mode 100644 index 524b4ed5cd9..00000000000 --- a/libs/dyn/visit_get_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestGetWithEmptyPath(t *testing.T) { - // An empty path means to return the value itself. - vin := dyn.V(42) - vout, err := dyn.GetByPath(vin, dyn.NewPath()) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} - -func TestGetOnNilValue(t *testing.T) { - var err error - _, err = dyn.GetByPath(dyn.NilValue, dyn.NewPath(dyn.Key("foo"))) - assert.ErrorContains(t, err, `expected a map to index "foo", found nil`) - _, err = dyn.GetByPath(dyn.NilValue, dyn.NewPath(dyn.Index(42))) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found nil`) -} - -func TestGetOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var err error - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Index(42))) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found map`) - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Key("baz"))) - assert.True(t, dyn.IsNoSuchKeyError(err)) - // "baz" is one edit away from "bar", so the error suggests it. - assert.ErrorContains(t, err, `key not found at "baz", did you mean "bar"?`) - - // A key that is close to no existing key gets no suggestion. - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Key("completely_different"))) - assert.True(t, dyn.IsNoSuchKeyError(err)) - assert.ErrorContains(t, err, `key not found at "completely_different"`) - assert.NotContains(t, err.Error(), "did you mean") - - vfoo, err := dyn.GetByPath(vin, dyn.NewPath(dyn.Key("foo"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V(42), vfoo) - - vbar, err := dyn.GetByPath(vin, dyn.NewPath(dyn.Key("bar"))) - assert.NoError(t, err) - assert.Equal(t, dyn.V(43), vbar) -} - -func TestGetOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - var err error - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Key("foo"))) - assert.ErrorContains(t, err, `expected a map to index "foo", found sequence`) - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Index(-1))) - assert.True(t, dyn.IsIndexOutOfBoundsError(err)) - assert.ErrorContains(t, err, `index out of bounds at "[-1]"`) - - _, err = dyn.GetByPath(vin, dyn.NewPath(dyn.Index(2))) - assert.True(t, dyn.IsIndexOutOfBoundsError(err)) - assert.ErrorContains(t, err, `index out of bounds at "[2]"`) - - v0, err := dyn.GetByPath(vin, dyn.NewPath(dyn.Index(0))) - assert.NoError(t, err) - assert.Equal(t, dyn.V(42), v0) - - v1, err := dyn.GetByPath(vin, dyn.NewPath(dyn.Index(1))) - assert.NoError(t, err) - assert.Equal(t, dyn.V(43), v1) -} diff --git a/libs/dyn/visit_map.go b/libs/dyn/visit_map.go deleted file mode 100644 index a20655e5c21..00000000000 --- a/libs/dyn/visit_map.go +++ /dev/null @@ -1,89 +0,0 @@ -package dyn - -import ( - "fmt" - "slices" -) - -// MapFunc is a function that maps a value to another value. -type MapFunc func(Path, Value) (Value, error) - -// Foreach returns a [MapFunc] that applies the specified [MapFunc] to each -// value in a map or sequence and returns the new map or sequence. -// If the input is nil, it returns nil. -func Foreach(fn MapFunc) MapFunc { - return func(p Path, v Value) (Value, error) { - switch v.Kind() { - case KindNil: - return v, nil - case KindMap: - m := v.MustMap().Clone() - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - nv, err := fn(p.Append(Key(pk.MustString())), pv) - if err != nil { - return InvalidValue, err - } - m.SetLoc(pk.MustString(), pk.Locations(), nv) - } - return NewValue(m, v.Locations()), nil - case KindSequence: - s := slices.Clone(v.MustSequence()) - for i, value := range s { - var err error - s[i], err = fn(p.Append(Index(i)), value) - if err != nil { - return InvalidValue, err - } - } - return NewValue(s, v.Locations()), nil - default: - return InvalidValue, fmt.Errorf("expected a map or sequence, found %s", v.Kind()) - } - } -} - -// Map applies a function to the value at the given path in the given value. -// It is identical to [MapByPath], except that it takes a string path instead of a [Path]. -func Map(v Value, path string, fn MapFunc) (Value, error) { - p, err := NewPathFromString(path) - if err != nil { - return InvalidValue, err - } - return MapByPath(v, p, fn) -} - -// MapByPath applies a function to the value at the given path in the given value. -// It is identical to [MapByPattern], except that it takes a [Path] instead of a [Pattern]. -// This means it only matches a single value, not a pattern of values. -func MapByPath(v Value, p Path, fn MapFunc) (Value, error) { - return MapByPattern(v, NewPatternFromPath(p), fn) -} - -// MapByPattern applies a function to the values whose paths match the given pattern in the given value. -// If successful, it returns the new value with all intermediate values copied and updated. -// -// If the pattern contains a key that doesn't exist, or an index that is out of bounds, -// it returns the original value and no error. -// -// If the pattern is invalid for the given value, it returns InvalidValue and an error. -func MapByPattern(v Value, p Pattern, fn MapFunc) (Value, error) { - nv, err := visit(v, EmptyPath, p, visitOptions{ - fn: fn, - }) - - // Check for success. - if err == nil { - return nv, nil - } - - // Return original value if: - // - any map or sequence is a nil, or - // - a key or index is missing - if IsCannotTraverseNilError(err) || IsNoSuchKeyError(err) || IsIndexOutOfBoundsError(err) { - return v, nil - } - - return nv, err -} diff --git a/libs/dyn/visit_map_test.go b/libs/dyn/visit_map_test.go deleted file mode 100644 index bff4b0bd3f9..00000000000 --- a/libs/dyn/visit_map_test.go +++ /dev/null @@ -1,477 +0,0 @@ -package dyn_test - -import ( - "errors" - "fmt" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMapWithEmptyPath(t *testing.T) { - // An empty path means to return the value itself. - vin := dyn.V(42) - vout, err := dyn.MapByPath(dyn.InvalidValue, dyn.EmptyPath, func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return vin, nil - }) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} - -func TestMapOnNilValue(t *testing.T) { - var nv dyn.Value - var err error - nv, err = dyn.MapByPath(dyn.NilValue, dyn.NewPath(dyn.Key("foo")), nil) - assert.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) - nv, err = dyn.MapByPath(dyn.NilValue, dyn.NewPath(dyn.Index(42)), nil) - assert.NoError(t, err) - assert.Equal(t, dyn.NilValue, nv) -} - -func TestMapFuncOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var err error - - _, err = dyn.MapByPath(vin, dyn.NewPath(dyn.Index(42)), nil) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found map`) - - // A key that does not exist is not an error. - vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("baz")), nil) - assert.NoError(t, err) - assert.Equal(t, vin, vout) - - // Note: in the test cases below we implicitly test that the original - // value is not modified as we repeatedly set values on it. - vfoo, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("foo")), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Key("foo")), p) - assert.Equal(t, dyn.V(42), v) - return dyn.V(44), nil - }) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 44, - "bar": 43, - }, vfoo.AsAny()) - - vbar, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("bar")), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Key("bar")), p) - assert.Equal(t, dyn.V(43), v) - return dyn.V(45), nil - }) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 42, - "bar": 45, - }, vbar.AsAny()) - - // Return error from map function. - ref := errors.New("error") - verr, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("foo")), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - }) - assert.Equal(t, dyn.InvalidValue, verr) - assert.ErrorIs(t, err, ref) -} - -func TestMapFuncOnMapWithEmptySequence(t *testing.T) { - variants := []dyn.Value{ - // empty sequence - dyn.V([]dyn.Value{}), - // non-empty sequence - dyn.V([]dyn.Value{dyn.V(42)}), - } - - for i := range variants { - vin := dyn.V(map[string]dyn.Value{ - "key": variants[i], - }) - - for j := range variants { - vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("key")), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return variants[j], nil - }) - assert.NoError(t, err) - assert.Equal(t, variants[j], vout.Get("key")) - } - } -} - -func TestMapFuncOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - var err error - - _, err = dyn.MapByPath(vin, dyn.NewPath(dyn.Key("foo")), nil) - assert.ErrorContains(t, err, `expected a map to index "foo", found sequence`) - - // An index that does not exist is not an error. - vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(2)), nil) - assert.NoError(t, err) - assert.Equal(t, vin, vout) - - // Note: in the test cases below we implicitly test that the original - // value is not modified as we repeatedly set values on it. - v0, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(0)), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Index(0)), p) - assert.Equal(t, dyn.V(42), v) - return dyn.V(44), nil - }) - assert.NoError(t, err) - assert.Equal(t, []any{44, 43}, v0.AsAny()) - - v1, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(1)), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Index(1)), p) - assert.Equal(t, dyn.V(43), v) - return dyn.V(45), nil - }) - assert.NoError(t, err) - assert.Equal(t, []any{42, 45}, v1.AsAny()) - - // Return error from map function. - ref := errors.New("error") - verr, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(0)), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - }) - assert.Equal(t, dyn.InvalidValue, verr) - assert.ErrorIs(t, err, ref) -} - -func TestMapFuncOnSequenceWithEmptySequence(t *testing.T) { - variants := []dyn.Value{ - // empty sequence - dyn.V([]dyn.Value{}), - // non-empty sequence - dyn.V([]dyn.Value{dyn.V(42)}), - } - - for i := range variants { - vin := dyn.V([]dyn.Value{ - variants[i], - }) - - for j := range variants { - vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(0)), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return variants[j], nil - }) - assert.NoError(t, err) - assert.Equal(t, variants[j], vout.Index(0)) - } - } -} - -func TestMapForeachOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var err error - - // Run foreach, adding 1 to each of the elements. - vout, err := dyn.Map(vin, ".", dyn.Foreach(func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - i, ok := v.AsInt() - require.True(t, ok, "expected an integer") - switch p[0].Key() { - case "foo": - assert.EqualValues(t, 42, i) - return dyn.V(43), nil - case "bar": - assert.EqualValues(t, 43, i) - return dyn.V(44), nil - default: - return dyn.InvalidValue, fmt.Errorf("unexpected key %q", p[0].Key()) - } - })) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 43, - "bar": 44, - }, vout.AsAny()) - - // Check that the original has not been modified. - assert.Equal(t, map[string]any{ - "foo": 42, - "bar": 43, - }, vin.AsAny()) -} - -func TestMapForeachOnMapError(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - // Check that an error from the map function propagates. - ref := errors.New("error") - _, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - })) - assert.ErrorIs(t, err, ref) -} - -func TestMapForeachOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - var err error - - // Run foreach, adding 1 to each of the elements. - vout, err := dyn.Map(vin, ".", dyn.Foreach(func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - i, ok := v.AsInt() - require.True(t, ok, "expected an integer") - switch p[0].Index() { - case 0: - assert.EqualValues(t, 42, i) - return dyn.V(43), nil - case 1: - assert.EqualValues(t, 43, i) - return dyn.V(44), nil - default: - return dyn.InvalidValue, fmt.Errorf("unexpected index %d", p[0].Index()) - } - })) - assert.NoError(t, err) - assert.Equal(t, []any{43, 44}, vout.AsAny()) - - // Check that the original has not been modified. - assert.Equal(t, []any{42, 43}, vin.AsAny()) -} - -func TestMapForeachOnSequenceError(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - // Check that an error from the map function propagates. - ref := errors.New("error") - _, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - })) - assert.ErrorIs(t, err, ref) -} - -func TestMapForeachOnOtherError(t *testing.T) { - vin := dyn.V(42) - - // Check that if foreach is applied to something other than a map or a sequence, it returns an error. - _, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, nil - })) - assert.ErrorContains(t, err, "expected a map or sequence, found int") -} - -func TestMapForeachOnNil(t *testing.T) { - vin := dyn.NilValue - - // Check that if foreach is applied to nil, it returns nil. - vout, err := dyn.Map(vin, ".", dyn.Foreach(func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, nil - })) - assert.NoError(t, err) - assert.Equal(t, dyn.NilValue, vout) -} - -func TestMapByPatternOnNilValue(t *testing.T) { - var err error - _, err = dyn.MapByPattern(dyn.NilValue, dyn.NewPattern(dyn.AnyKey()), nil) - assert.ErrorContains(t, err, `expected a map at "", found nil`) - _, err = dyn.MapByPattern(dyn.NilValue, dyn.NewPattern(dyn.AnyIndex()), nil) - assert.ErrorContains(t, err, `expected a sequence at "", found nil`) -} - -func TestMapByPatternOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V(42), - }), - "b": dyn.V(map[string]dyn.Value{ - "c": dyn.V(43), - }), - }) - - var err error - - // Expect an error if the pattern structure doesn't match the value structure. - _, err = dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey(), dyn.Index(0)), nil) - assert.ErrorContains(t, err, `expected a sequence to index`) - - // Apply function to pattern "*.b". - vout, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey(), dyn.Key("b")), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Key("a"), dyn.Key("b")), p) - assert.Equal(t, dyn.V(42), v) - return dyn.V(44), nil - }) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "a": map[string]any{ - "b": 44, - }, - "b": map[string]any{ - "c": 43, - }, - }, vout.AsAny()) -} - -func TestMapByPatternOnMapWithoutMatch(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V(42), - }), - "b": dyn.V(map[string]dyn.Value{ - "c": dyn.V(43), - }), - }) - - // Apply function to pattern "*.zzz". - vout, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey(), dyn.Key("zzz")), nil) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} - -func TestMapByPatternOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V(42), - }), - dyn.V([]dyn.Value{ - dyn.V(43), - dyn.V(44), - }), - }) - - var err error - - // Expect an error if the pattern structure doesn't match the value structure. - _, err = dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyIndex(), dyn.Key("a")), nil) - assert.ErrorContains(t, err, `expected a map to index`) - - // Apply function to pattern "*.c". - vout, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyIndex(), dyn.Index(1)), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - assert.Equal(t, dyn.NewPath(dyn.Index(1), dyn.Index(1)), p) - assert.Equal(t, dyn.V(44), v) - return dyn.V(45), nil - }) - assert.NoError(t, err) - assert.Equal(t, []any{ - []any{42}, - []any{43, 45}, - }, vout.AsAny()) -} - -func TestMapByPatternWildcardSkipsNonMatchingSiblings(t *testing.T) { - tests := []struct { - name string - vin dyn.Value - pattern dyn.Pattern - want any - visited []string - }{ - { - name: "nil sibling under any key", - vin: dyn.V(map[string]dyn.Value{ - "a": dyn.NilValue, - "b": dyn.V(map[string]dyn.Value{"x": dyn.V(1)}), - }), - pattern: dyn.NewPattern(dyn.AnyKey(), dyn.Key("x")), - want: map[string]any{ - "a": nil, - "b": map[string]any{"x": 42}, - }, - visited: []string{"b.x"}, - }, - { - name: "nil sequence sibling under any key", - vin: dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{"tasks": dyn.NilValue}), - "b": dyn.V(map[string]dyn.Value{"tasks": dyn.V([]dyn.Value{dyn.V(1)})}), - }), - pattern: dyn.NewPattern(dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex()), - want: map[string]any{ - "a": map[string]any{"tasks": nil}, - "b": map[string]any{"tasks": []any{42}}, - }, - visited: []string{"b.tasks[0]"}, - }, - { - name: "wrong kind sibling under any key", - vin: dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{"tasks": dyn.V("oops")}), - "b": dyn.V(map[string]dyn.Value{"tasks": dyn.V([]dyn.Value{dyn.V(1)})}), - }), - pattern: dyn.NewPattern(dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex()), - want: map[string]any{ - "a": map[string]any{"tasks": "oops"}, - "b": map[string]any{"tasks": []any{42}}, - }, - visited: []string{"b.tasks[0]"}, - }, - { - name: "nil element under any index", - vin: dyn.V([]dyn.Value{ - dyn.NilValue, - dyn.V(map[string]dyn.Value{"x": dyn.V(1)}), - }), - pattern: dyn.NewPattern(dyn.AnyIndex(), dyn.Key("x")), - want: []any{ - nil, - map[string]any{"x": 42}, - }, - visited: []string{"[1].x"}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - var visited []string - vout, err := dyn.MapByPattern(tc.vin, tc.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - visited = append(visited, p.String()) - return dyn.V(42), nil - }) - assert.NoError(t, err) - assert.Equal(t, tc.want, vout.AsAny()) - assert.Equal(t, tc.visited, visited) - }) - } -} - -func TestMapByPatternWildcardPropagatesMapFuncError(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{"x": dyn.V(1)}), - }) - - ref := errors.New("error") - _, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey(), dyn.Key("x")), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { - return dyn.InvalidValue, ref - }) - assert.ErrorIs(t, err, ref) -} - -func TestMapByPatternOnSequenceWithoutMatch(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V(42), - }), - dyn.V([]dyn.Value{ - dyn.V(43), - dyn.V(44), - }), - }) - - // Apply function to pattern "*.zzz". - vout, err := dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyIndex(), dyn.Index(42)), nil) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} diff --git a/libs/dyn/visit_set.go b/libs/dyn/visit_set.go deleted file mode 100644 index 1b3c7f1b661..00000000000 --- a/libs/dyn/visit_set.go +++ /dev/null @@ -1,77 +0,0 @@ -package dyn - -import ( - "fmt" - "slices" -) - -// Set assigns a new value at the specified path in the specified value. -// It is identical to [SetByPath], except that it takes a string path instead of a [Path]. -func Set(v Value, path string, nv Value) (Value, error) { - p, err := NewPathFromString(path) - if err != nil { - return InvalidValue, err - } - return SetByPath(v, p, nv) -} - -// SetByPath assigns the given value at the specified path in the specified value. -// If successful, it returns the new value with all intermediate values copied and updated. -// If the path doesn't exist, it returns InvalidValue and an error. -func SetByPath(v Value, p Path, nv Value) (Value, error) { - lp := len(p) - if lp == 0 { - return nv, nil - } - - component := p[lp-1] - p = p[:lp-1] - - return visit(v, EmptyPath, NewPatternFromPath(p), visitOptions{ - fn: func(prefix Path, v Value) (Value, error) { - path := append(prefix, component) - - switch { - case component.isKey(): - // Expect a map to be set if this is a key. - m, ok := v.AsMap() - if !ok { - return InvalidValue, fmt.Errorf("expected a map to index %q, found %s", path, v.Kind()) - } - - // Return an updated map value. - m = m.Clone() - m.SetLoc(component.key, nil, nv) - return Value{ - v: m, - k: KindMap, - l: v.l, - }, nil - - case component.isIndex(): - // Expect a sequence to be set if this is an index. - s, ok := v.AsSequence() - if !ok { - return InvalidValue, fmt.Errorf("expected a sequence to index %q, found %s", path, v.Kind()) - } - - // Lookup current value in the sequence. - if component.index < 0 || component.index >= len(s) { - return InvalidValue, indexOutOfBoundsError{prefix} - } - - // Return an updated sequence value. - s = slices.Clone(s) - s[component.index] = nv - return Value{ - v: s, - k: KindSequence, - l: v.l, - }, nil - - default: - panic("invalid component") - } - }, - }) -} diff --git a/libs/dyn/visit_set_test.go b/libs/dyn/visit_set_test.go deleted file mode 100644 index b3847158757..00000000000 --- a/libs/dyn/visit_set_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestSetWithEmptyPath(t *testing.T) { - // An empty path means to return the value itself. - vin := dyn.V(42) - vout, err := dyn.SetByPath(dyn.InvalidValue, dyn.EmptyPath, vin) - assert.NoError(t, err) - assert.Equal(t, vin, vout) -} - -func TestSetOnNilValue(t *testing.T) { - var err error - _, err = dyn.SetByPath(dyn.NilValue, dyn.NewPath(dyn.Key("foo")), dyn.V(42)) - assert.ErrorContains(t, err, `expected a map to index "foo", found nil`) - _, err = dyn.SetByPath(dyn.NilValue, dyn.NewPath(dyn.Index(42)), dyn.V(42)) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found nil`) -} - -func TestSetOnMap(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var err error - - _, err = dyn.SetByPath(vin, dyn.NewPath(dyn.Index(42)), dyn.V(42)) - assert.ErrorContains(t, err, `expected a sequence to index "[42]", found map`) - - // Note: in the test cases below we implicitly test that the original - // value is not modified as we repeatedly set values on it. - - vfoo, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Key("foo")), dyn.V(44)) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 44, - "bar": 43, - }, vfoo.AsAny()) - - vbar, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Key("bar")), dyn.V(45)) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 42, - "bar": 45, - }, vbar.AsAny()) - - vbaz, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Key("baz")), dyn.V(46)) - assert.NoError(t, err) - assert.Equal(t, map[string]any{ - "foo": 42, - "bar": 43, - "baz": 46, - }, vbaz.AsAny()) -} - -func TestSetOnSequence(t *testing.T) { - vin := dyn.V([]dyn.Value{ - dyn.V(42), - dyn.V(43), - }) - - var err error - - _, err = dyn.SetByPath(vin, dyn.NewPath(dyn.Key("foo")), dyn.V(42)) - assert.ErrorContains(t, err, `expected a map to index "foo", found sequence`) - - // It is not allowed to set a value at an index that is out of bounds. - _, err = dyn.SetByPath(vin, dyn.NewPath(dyn.Index(-1)), dyn.V(42)) - assert.True(t, dyn.IsIndexOutOfBoundsError(err)) - _, err = dyn.SetByPath(vin, dyn.NewPath(dyn.Index(2)), dyn.V(42)) - assert.True(t, dyn.IsIndexOutOfBoundsError(err)) - - // Note: in the test cases below we implicitly test that the original - // value is not modified as we repeatedly set values on it. - - v0, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Index(0)), dyn.V(44)) - assert.NoError(t, err) - assert.Equal(t, []any{44, 43}, v0.AsAny()) - - v1, err := dyn.SetByPath(vin, dyn.NewPath(dyn.Index(1)), dyn.V(45)) - assert.NoError(t, err) - assert.Equal(t, []any{42, 45}, v1.AsAny()) -} diff --git a/libs/dyn/visit_test.go b/libs/dyn/visit_test.go deleted file mode 100644 index b54e6ad37c7..00000000000 --- a/libs/dyn/visit_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package dyn_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestVisitCallbackPathCopy(t *testing.T) { - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V(42), - "bar": dyn.V(43), - }) - - var paths []dyn.Path - - // The callback should receive a copy of the path. - // If the same underlying value is used, all collected paths will be the same. - // This test uses `MapByPattern` to collect all paths in the map. - // Visit itself doesn't have public functions and we exclusively use black-box testing for this package. - _, _ = dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - paths = append(paths, p) - return v, nil - }) - - // Verify that the paths retained their original values. - var strings []string - for _, p := range paths { - strings = append(strings, p.String()) - } - assert.ElementsMatch(t, strings, []string{ - "foo", - "bar", - }) -} diff --git a/libs/dyn/walk.go b/libs/dyn/walk.go deleted file mode 100644 index 08123cf85e9..00000000000 --- a/libs/dyn/walk.go +++ /dev/null @@ -1,95 +0,0 @@ -package dyn - -import "errors" - -// WalkValueFunc is the type of the function called by Walk to traverse the configuration tree. -type WalkValueFunc func(p Path, v Value) (Value, error) - -// ErrDrop may be returned by WalkValueFunc to remove a value from the subtree. -var ErrDrop = errors.New("drop value from subtree") - -// ErrSkip may be returned by WalkValueFunc to skip traversal of a subtree. -var ErrSkip = errors.New("skip traversal of subtree") - -// Walk walks the configuration tree and calls the given function on each node. -// The callback may return ErrDrop to remove a value from the subtree. -// The callback may return ErrSkip to skip traversal of a subtree. -// If the callback returns another error, the walk is aborted, and the error is returned. -func Walk(v Value, fn func(p Path, v Value) (Value, error)) (Value, error) { - return walk(v, EmptyPath, fn) -} - -// Unexported counterpart to Walk. -// It carries the path leading up to the current node, -// such that it can be passed to the WalkValueFunc. -func walk(v Value, p Path, fn func(p Path, v Value) (Value, error)) (Value, error) { - v, err := fn(p, v) - if err != nil { - if errors.Is(err, ErrSkip) { - return v, nil - } - return InvalidValue, err - } - - switch v.Kind() { - case KindMap: - m := v.MustMap() - out := newMappingWithSize(m.Len()) - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - nv, err := walk(pv, append(p, Key(pk.MustString())), fn) - if errors.Is(err, ErrDrop) { - continue - } - if err != nil { - return InvalidValue, err - } - out.SetLoc(pk.MustString(), pk.Locations(), nv) - } - v.v = out - case KindSequence: - s := v.MustSequence() - out := make([]Value, 0, len(s)) - for i := range s { - nv, err := walk(s[i], append(p, Index(i)), fn) - if errors.Is(err, ErrDrop) { - continue - } - if err != nil { - return InvalidValue, err - } - out = append(out, nv) - } - v.v = out - default: - // Other kinds are not walkable. - // This was a leaf node. - } - - return v, nil -} - -// CollectLeafPaths traverses the value and returns all paths (as dot notation strings) to leaf nodes (non-map, non-sequence). -// The return value is not ordered. -func CollectLeafPaths(v Value) []string { - var paths []string - - Walk(v, func(p Path, v Value) (Value, error) { //nolint:errcheck - if len(p) == 0 { - return v, nil - } - - switch v.Kind() { - case KindMap, KindSequence: - // Ignore internal nodes. - default: - // This was a leaf node. - paths = append(paths, p.String()) - } - - return v, nil - }) - - return paths -} diff --git a/libs/dyn/walk_read_only.go b/libs/dyn/walk_read_only.go deleted file mode 100644 index 8f8d15e06ea..00000000000 --- a/libs/dyn/walk_read_only.go +++ /dev/null @@ -1,46 +0,0 @@ -package dyn - -import "errors" - -// WalkReadOnly walks the configuration tree in readonly mode and calls the given function on each node. -// The callback may return ErrSkip to skip traversal of a subtree. -// If the callback returns another error, the walk is aborted, and the error is returned. -func WalkReadOnly(v Value, fn func(p Path, v Value) error) error { - return walkReadOnly(v, EmptyPath, fn) -} - -// Unexported counterpart to WalkReadOnly. -// It carries the path leading up to the current node, -// such that it can be passed to the callback function. -func walkReadOnly(v Value, p Path, fn func(p Path, v Value) error) error { - if err := fn(p, v); err != nil { - if errors.Is(err, ErrSkip) { - return nil - } - return err - } - - switch v.Kind() { - case KindMap: - m := v.MustMap() - for _, pair := range m.Pairs() { - pk := pair.Key - pv := pair.Value - if err := walkReadOnly(pv, append(p, Key(pk.MustString())), fn); err != nil { - return err - } - } - case KindSequence: - s := v.MustSequence() - for i := range s { - if err := walkReadOnly(s[i], append(p, Index(i)), fn); err != nil { - return err - } - } - default: - // Other kinds are not walkable. - // This was a leaf node. - } - - return nil -} diff --git a/libs/dyn/walk_read_only_test.go b/libs/dyn/walk_read_only_test.go deleted file mode 100644 index f6df5344d8c..00000000000 --- a/libs/dyn/walk_read_only_test.go +++ /dev/null @@ -1,245 +0,0 @@ -package dyn_test - -import ( - "errors" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestWalkReadOnly(t *testing.T) { - tests := []struct { - name string - input dyn.Value - expectedPaths []dyn.Path - expectedValues []dyn.Value - }{ - { - name: "simple map", - input: dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("a"), Value: dyn.V("1")}, - {Key: dyn.V("b"), Value: dyn.V("2")}, - }, - )), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("b")}, - }, - expectedValues: []dyn.Value{ - dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("a"), Value: dyn.V("1")}, - {Key: dyn.V("b"), Value: dyn.V("2")}, - }, - )), - dyn.V("1"), - dyn.V("2"), - }, - }, - { - name: "nested map", - input: dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("a"), dyn.Key("b")}, - }, - expectedValues: []dyn.Value{ - dyn.V(map[string]dyn.Value{ - "a": dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - dyn.V("1"), - }, - }, - { - name: "sequence", - input: dyn.V([]dyn.Value{ - dyn.V("1"), - dyn.V("2"), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Index(0)}, - {dyn.Index(1)}, - }, - expectedValues: []dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V("1"), - dyn.V("2"), - }), - dyn.V("1"), - dyn.V("2"), - }, - }, - { - name: "nested sequence", - input: dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V("1"), - }), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Index(0)}, - {dyn.Index(0), dyn.Index(0)}, - }, - expectedValues: []dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V([]dyn.Value{ - dyn.V("1"), - }), - }), - dyn.V([]dyn.Value{ - dyn.V("1"), - }), - dyn.V("1"), - }, - }, - { - name: "complex structure", - input: dyn.V(map[string]dyn.Value{ - "a": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - }), - expectedPaths: []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("a"), dyn.Index(0)}, - {dyn.Key("a"), dyn.Index(0), dyn.Key("b")}, - }, - expectedValues: []dyn.Value{ - dyn.V(map[string]dyn.Value{ - "a": dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - }), - dyn.V([]dyn.Value{ - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - }), - dyn.V(map[string]dyn.Value{ - "b": dyn.V("1"), - }), - dyn.V("1"), - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - visitedPaths := make([]dyn.Path, 0, len(tt.expectedPaths)) - visitedValues := make([]dyn.Value, 0, len(tt.expectedValues)) - - err := dyn.WalkReadOnly(tt.input, func(p dyn.Path, v dyn.Value) error { - visitedPaths = append(visitedPaths, p) - visitedValues = append(visitedValues, v) - return nil - }) - assert.NoError(t, err) - assert.Equal(t, tt.expectedPaths, visitedPaths) - assert.Equal(t, tt.expectedValues, visitedValues) - }) - } -} - -func TestWalkReadOnly_Error(t *testing.T) { - testErr := errors.New("test error") - input := dyn.V(map[string]dyn.Value{ - "a": dyn.V("1"), - }) - - err := dyn.WalkReadOnly(input, func(p dyn.Path, v dyn.Value) error { - if p.Equal(dyn.Path{dyn.Key("a")}) { - return testErr - } - return nil - }) - - assert.Equal(t, err, testErr) -} - -func TestWalkReadOnly_SkipPaths(t *testing.T) { - va := dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("b"), Value: dyn.V("1")}, - {Key: dyn.V("c"), Value: dyn.V("2")}, - }, - )) - - vd := dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - {Key: dyn.V("e"), Value: dyn.V("3")}, - }, - )) - - input := dyn.V(dyn.NewMappingFromPairs( - []dyn.Pair{ - { - Key: dyn.V("a"), - Value: va, - }, - { - Key: dyn.V("d"), - Value: vd, - }, - { - Key: dyn.V("f"), - Value: dyn.V("4"), - }, - }, - )) - - skipPaths := map[string]bool{ - "a.b": true, - "d": true, - } - - var visitedPaths []dyn.Path - var visitedValues []dyn.Value - - err := dyn.WalkReadOnly(input, func(p dyn.Path, v dyn.Value) error { - _, ok := skipPaths[p.String()] - if ok { - return dyn.ErrSkip - } - - visitedPaths = append(visitedPaths, p) - visitedValues = append(visitedValues, v) - return nil - }) - assert.NoError(t, err) - - expectedPaths := []dyn.Path{ - dyn.EmptyPath, - {dyn.Key("a")}, - {dyn.Key("a"), dyn.Key("c")}, - {dyn.Key("f")}, - } - expectedValues := []dyn.Value{ - input, - va, - dyn.V("2"), - dyn.V("4"), - } - - assert.Equal(t, expectedPaths, visitedPaths) - assert.Equal(t, expectedValues, visitedValues) -} diff --git a/libs/dyn/walk_test.go b/libs/dyn/walk_test.go deleted file mode 100644 index f59683f0381..00000000000 --- a/libs/dyn/walk_test.go +++ /dev/null @@ -1,269 +0,0 @@ -package dyn_test - -import ( - "errors" - "testing" - - . "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Return values for specific paths. -type walkReturn struct { - path Path - - // Return values. - fn func(Value) Value - err error -} - -// Track the calls to the callback. -type walkCall struct { - path Path - value Value -} - -// Track the calls to the callback. -type walkCallTracker struct { - returns []walkReturn - calls []walkCall -} - -func (w *walkCallTracker) on(path string, fn func(Value) Value, err error) { - w.returns = append(w.returns, walkReturn{MustPathFromString(path), fn, err}) -} - -func (w *walkCallTracker) returnSkip(path string) { - w.on(path, func(v Value) Value { return v }, ErrSkip) -} - -func (w *walkCallTracker) returnDrop(path string) { - w.on(path, func(v Value) Value { return InvalidValue }, ErrDrop) -} - -func (w *walkCallTracker) track(p Path, v Value) (Value, error) { - w.calls = append(w.calls, walkCall{p, v}) - - // Look for matching return. - for _, r := range w.returns { - if p.Equal(r.path) { - return r.fn(v), r.err - } - } - - return v, nil -} - -func TestWalkEmpty(t *testing.T) { - var tracker walkCallTracker - - value := V(nil) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal(t, value, out) - - // The callback should have been called once. - assert.Len(t, tracker.calls, 1) - - // The call should have been made with the empty path. - assert.Equal(t, EmptyPath, tracker.calls[0].path) - - // The value should be the same as the input. - assert.Equal(t, value, tracker.calls[0].value) -} - -func TestWalkMapSkip(t *testing.T) { - var tracker walkCallTracker - - // Skip traversal of the root value. - tracker.returnSkip(".") - - value := V(map[string]Value{ - "key": V("value"), - }) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal( - t, - V(map[string]Value{ - "key": V("value"), - }), - out, - ) - - // The callback should have been called once. - assert.Len(t, tracker.calls, 1) - - // The call should have been made with the empty path. - assert.Equal(t, EmptyPath, tracker.calls[0].path) - - // The value should be the same as the input. - assert.Equal(t, value, tracker.calls[0].value) -} - -func TestWalkMapDrop(t *testing.T) { - var tracker walkCallTracker - - // Drop the value at key "foo". - tracker.returnDrop(".foo") - - value := V(map[string]Value{ - "foo": V("bar"), - "bar": V("baz"), - }) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal( - t, - V(map[string]Value{ - "bar": V("baz"), - }), - out, - ) - - // The callback should have been called for the root and every key in the map. - assert.Len(t, tracker.calls, 3) - - // Calls 2 and 3 have been made for the keys in the map. - assert.ElementsMatch(t, - []Path{ - tracker.calls[1].path, - tracker.calls[2].path, - }, []Path{ - MustPathFromString(".foo"), - MustPathFromString(".bar"), - }) -} - -func TestWalkMapError(t *testing.T) { - var tracker walkCallTracker - - // Return an error from the callback for key "foo". - cerr := errors.New("error!") - tracker.on(".foo", func(v Value) Value { return v }, cerr) - - value := V(map[string]Value{ - "foo": V("bar"), - }) - out, err := Walk(value, tracker.track) - assert.Equal(t, cerr, err) - assert.Equal(t, InvalidValue, out) - - // The callback should have been called twice. - assert.Len(t, tracker.calls, 2) - - // The second call was for the value at key "foo". - assert.Equal(t, MustPathFromString(".foo"), tracker.calls[1].path) -} - -func TestWalkSequenceSkip(t *testing.T) { - var tracker walkCallTracker - - // Skip traversal of the root value. - tracker.returnSkip(".") - - value := V([]Value{ - V("foo"), - V("bar"), - }) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal( - t, - V([]Value{ - V("foo"), - V("bar"), - }), - out, - ) - - // The callback should have been called once. - assert.Len(t, tracker.calls, 1) - - // The call should have been made with the empty path. - assert.Equal(t, EmptyPath, tracker.calls[0].path) - - // The value should be the same as the input. - assert.Equal(t, value, tracker.calls[0].value) -} - -func TestWalkSequenceDrop(t *testing.T) { - var tracker walkCallTracker - - // Drop the value at index 1. - tracker.returnDrop(".[1]") - - value := V([]Value{ - V("foo"), - V("bar"), - V("baz"), - }) - out, err := Walk(value, tracker.track) - require.NoError(t, err) - assert.Equal( - t, - V([]Value{ - V("foo"), - V("baz"), - }), - out, - ) - - // The callback should have been called for the root and every value in the sequence. - assert.Len(t, tracker.calls, 4) - - // The second call was for the value at index 0. - assert.Equal(t, MustPathFromString(".[0]"), tracker.calls[1].path) - assert.Equal(t, V("foo"), tracker.calls[1].value) - - // The third call was for the value at index 1. - assert.Equal(t, MustPathFromString(".[1]"), tracker.calls[2].path) - assert.Equal(t, V("bar"), tracker.calls[2].value) - - // The fourth call was for the value at index 2. - assert.Equal(t, MustPathFromString(".[2]"), tracker.calls[3].path) - assert.Equal(t, V("baz"), tracker.calls[3].value) -} - -func TestWalkSequenceError(t *testing.T) { - var tracker walkCallTracker - - // Return an error from the callback for index 1. - cerr := errors.New("error!") - tracker.on(".[1]", func(v Value) Value { return v }, cerr) - - value := V([]Value{ - V("foo"), - V("bar"), - }) - out, err := Walk(value, tracker.track) - assert.Equal(t, cerr, err) - assert.Equal(t, InvalidValue, out) - - // The callback should have been called three times. - assert.Len(t, tracker.calls, 3) - - // The second call was for the value at index 0. - assert.Equal(t, MustPathFromString(".[0]"), tracker.calls[1].path) - assert.Equal(t, V("foo"), tracker.calls[1].value) - - // The third call was for the value at index 1. - assert.Equal(t, MustPathFromString(".[1]"), tracker.calls[2].path) - assert.Equal(t, V("bar"), tracker.calls[2].value) -} - -func TestCollectLeafPaths(t *testing.T) { - v := V(map[string]Value{ - "a": V(1), - "b": V(map[string]Value{ - "c": V(2), - "d": V(map[string]Value{ - "e": V(3), - }), - }), - "f": V([]Value{V(4), V(5)}), - }) - paths := CollectLeafPaths(v) - assert.ElementsMatch(t, []string{"a", "b.c", "b.d.e", "f[0]", "f[1]"}, paths) -} diff --git a/libs/dyn/yamlloader/loader.go b/libs/dyn/yamlloader/loader.go deleted file mode 100644 index aebbf0883e9..00000000000 --- a/libs/dyn/yamlloader/loader.go +++ /dev/null @@ -1,291 +0,0 @@ -package yamlloader - -import ( - "fmt" - "math" - "strconv" - "strings" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "go.yaml.in/yaml/v3" -) - -// LocationError is an error with a YAML source location that can be displayed -// to the user with a file path, line, and column number. -type LocationError struct { - Loc diag.Location - Summary string -} - -func (e *LocationError) Error() string { - return fmt.Sprintf("yaml (%s): %s", e.Loc, e.Summary) -} - -type loader struct { - path string - - // Aliases currently being expanded. A self-referential anchor (e.g. - // "a: &x {b: *x}") makes the yaml.v3 node graph cyclic; without this - // bookkeeping, expansion would recurse until stack overflow. - activeAliases map[*yaml.Node]bool -} - -func errorf(loc diag.Location, format string, args ...any) error { - return fmt.Errorf("yaml (%s): %s", loc, fmt.Sprintf(format, args...)) -} - -func newLoader(path string) *loader { - return &loader{ - path: path, - activeAliases: make(map[*yaml.Node]bool), - } -} - -func (d *loader) location(node *yaml.Node) diag.Location { - return diag.Location{ - File: d.path, - Line: node.Line, - Column: node.Column, - } -} - -func (d *loader) load(node *yaml.Node) (dyn.Value, error) { - loc := diag.Location{ - File: d.path, - Line: node.Line, - Column: node.Column, - } - - var value dyn.Value - var err error - - switch node.Kind { - case yaml.DocumentNode: - value, err = d.loadDocument(node, loc) - case yaml.SequenceNode: - value, err = d.loadSequence(node, loc) - case yaml.MappingNode: - value, err = d.loadMapping(node, loc) - case yaml.ScalarNode: - value, err = d.loadScalar(node, loc) - case yaml.AliasNode: - value, err = d.loadAlias(node, loc) - default: - return dyn.InvalidValue, errorf(loc, "unknown node kind: %v", node.Kind) - } - - if err != nil { - return value, err - } - - // Mark value as anchor if needed. - // If this node doesn't map to a type, we don't need to warn about it. - if node.Anchor != "" { - value = value.MarkAnchor() - } - - return value, nil -} - -func (d *loader) loadDocument(node *yaml.Node, loc diag.Location) (dyn.Value, error) { - return d.load(node.Content[0]) -} - -func (d *loader) loadSequence(node *yaml.Node, loc diag.Location) (dyn.Value, error) { - acc := make([]dyn.Value, len(node.Content)) - for i, n := range node.Content { - v, err := d.load(n) - if err != nil { - return dyn.InvalidValue, err - } - - acc[i] = v - } - - return dyn.NewValue(acc, []diag.Location{loc}), nil -} - -func (d *loader) loadMapping(node *yaml.Node, loc diag.Location) (dyn.Value, error) { - var merge *yaml.Node - - acc := dyn.NewMapping() - for i := 0; i < len(node.Content); i += 2 { - key := node.Content[i] - val := node.Content[i+1] - - // Assert that keys are strings - if key.Kind != yaml.ScalarNode { - return dyn.InvalidValue, errorf(loc, "key is not a scalar") - } - - st := key.ShortTag() - switch st { - case "!!str": - // OK - case "!!null": - // A literal unquoted "null" is treated as a null value by the YAML parser. - // However, when used as a key, it is treated as the string "null". - case "!!merge": - if merge != nil { - // The YAML merge key spec allows a single '<<' key per mapping. - // To merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]'. - return dyn.InvalidValue, &LocationError{ - Loc: d.location(key), - Summary: "duplicate YAML merge key ('<<') is not allowed; to merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]'", - } - } - merge = val - continue - default: - return dyn.InvalidValue, errorf(loc, "invalid key tag: %v", st) - } - - loc := []diag.Location{{ - File: d.path, - Line: key.Line, - Column: key.Column, - }} - - v, err := d.load(val) - if err != nil { - return dyn.InvalidValue, err - } - - acc.SetLoc(key.Value, loc, v) - } - - if merge == nil { - return dyn.NewValue(acc, []diag.Location{loc}), nil - } - - // Build location for the merge node. - mloc := d.location(merge) - merr := errorf(mloc, "map merge requires map or sequence of maps as the value") - - // Flatten the merge node into a slice of nodes. - // It can be either a single node or a sequence of nodes. - var mnodes []*yaml.Node - switch merge.Kind { - case yaml.SequenceNode: - mnodes = merge.Content - case yaml.AliasNode: - mnodes = []*yaml.Node{merge} - default: - return dyn.NilValue, merr - } - - // Build a sequence of values to merge. - // The entries that we already accumulated have precedence. - var seq []dyn.Mapping - for _, n := range mnodes { - v, err := d.load(n) - if err != nil { - return dyn.InvalidValue, err - } - m, ok := v.AsMap() - if !ok { - return dyn.NilValue, merr - } - seq = append(seq, m) - } - - // Append the accumulated entries to the sequence. - seq = append(seq, acc) - out := dyn.NewMapping() - for _, m := range seq { - out.Merge(m) - } - - return dyn.NewValue(out, []diag.Location{loc}), nil -} - -func newIntValue(i64 int64, loc diag.Location) dyn.Value { - // Use regular int type instead of int64 if possible. - if i64 >= math.MinInt32 && i64 <= math.MaxInt32 { - return dyn.NewValue(int(i64), []diag.Location{loc}) - } - return dyn.NewValue(i64, []diag.Location{loc}) -} - -func (d *loader) loadScalar(node *yaml.Node, loc diag.Location) (dyn.Value, error) { - st := node.ShortTag() - switch st { - case "!!str": - return dyn.NewValue(node.Value, []diag.Location{loc}), nil - case "!!bool": - switch strings.ToLower(node.Value) { - case "true": - return dyn.NewValue(true, []diag.Location{loc}), nil - case "false": - return dyn.NewValue(false, []diag.Location{loc}), nil - default: - return dyn.InvalidValue, errorf(loc, "invalid bool value: %v", node.Value) - } - case "!!int": - // Try to parse the an integer value in base 10. - // We trim leading zeros to avoid octal parsing of the "0" prefix. - // See "testdata/spec_example_2.19.yml" for background. - i64, err := strconv.ParseInt(strings.TrimLeft(node.Value, "0"), 10, 64) - if err == nil { - return newIntValue(i64, loc), nil - } - // Let the [ParseInt] function figure out the base. - i64, err = strconv.ParseInt(node.Value, 0, 64) - if err == nil { - return newIntValue(i64, loc), nil - } - return dyn.InvalidValue, errorf(loc, "invalid int value: %v", node.Value) - case "!!float": - f64, err := strconv.ParseFloat(node.Value, 64) - if err != nil { - // Deal with infinity prefixes. - v := strings.ToLower(node.Value) - switch { - case strings.HasPrefix(v, "+"): - v = strings.TrimPrefix(v, "+") - f64 = math.Inf(1) - case strings.HasPrefix(v, "-"): - v = strings.TrimPrefix(v, "-") - f64 = math.Inf(-1) - default: - // No prefix. - f64 = math.Inf(1) - } - - // Deal with infinity and NaN values. - switch v { - case ".inf": - return dyn.NewValue(f64, []diag.Location{loc}), nil - case ".nan": - return dyn.NewValue(math.NaN(), []diag.Location{loc}), nil - } - - return dyn.InvalidValue, errorf(loc, "invalid float value: %v", node.Value) - } - return dyn.NewValue(f64, []diag.Location{loc}), nil - case "!!null": - return dyn.NewValue(nil, []diag.Location{loc}), nil - case "!!timestamp": - t, err := dyn.NewTime(node.Value) - if err == nil { - return dyn.NewValue(t, []diag.Location{loc}), nil - } - return dyn.InvalidValue, errorf(loc, "invalid timestamp value: %v", node.Value) - default: - return dyn.InvalidValue, errorf(loc, "unknown tag: %v", st) - } -} - -func (d *loader) loadAlias(node *yaml.Node, loc diag.Location) (dyn.Value, error) { - if d.activeAliases[node] { - return dyn.InvalidValue, errorf(loc, "cyclic reference to anchor %q", node.Value) - } - - // The same alias node may be reached again through another alias to an - // enclosing anchor, which is not a cycle. - d.activeAliases[node] = true - defer delete(d.activeAliases, node) - - return d.load(node.Alias) -} diff --git a/libs/dyn/yamlloader/testdata/anchor_01.yml b/libs/dyn/yamlloader/testdata/anchor_01.yml deleted file mode 100644 index b677d60b808..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_01.yml +++ /dev/null @@ -1,12 +0,0 @@ -# 1. Basic Anchor and Alias -defaults: &DEFAULTS - color: red - size: large - -shirt1: - <<: *DEFAULTS - pattern: striped - -shirt2: - <<: *DEFAULTS - pattern: plain diff --git a/libs/dyn/yamlloader/testdata/anchor_02.yml b/libs/dyn/yamlloader/testdata/anchor_02.yml deleted file mode 100644 index 86650987f3d..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_02.yml +++ /dev/null @@ -1,13 +0,0 @@ -# 2. Merging Anchors -# Here, multiple anchors can be merged into a single item. -defaults: &DEFAULTS - color: red - size: large - -materials: &MATERIALS - primary: cotton - secondary: polyester - -shirt: - <<: [*DEFAULTS, *MATERIALS] - pattern: striped diff --git a/libs/dyn/yamlloader/testdata/anchor_03.yml b/libs/dyn/yamlloader/testdata/anchor_03.yml deleted file mode 100644 index f06b7999ef0..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_03.yml +++ /dev/null @@ -1,10 +0,0 @@ -# 3. Overriding Merged Anchor Values -# You can override values when merging. -defaults: &DEFAULTS - color: red - size: large - pattern: plain - -shirt: - <<: *DEFAULTS - color: blue diff --git a/libs/dyn/yamlloader/testdata/anchor_04.yml b/libs/dyn/yamlloader/testdata/anchor_04.yml deleted file mode 100644 index bbc4bc02707..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_04.yml +++ /dev/null @@ -1,16 +0,0 @@ -# 4. Nested Anchors -# This demonstrates the reuse of nested content. -address: &ADDRESS - city: San Francisco - state: CA - country: USA - -person1: - name: Alice - address: *ADDRESS - -person2: - name: Bob - address: - <<: *ADDRESS - city: Los Angeles diff --git a/libs/dyn/yamlloader/testdata/anchor_05.yml b/libs/dyn/yamlloader/testdata/anchor_05.yml deleted file mode 100644 index 9a4c961b7c8..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_05.yml +++ /dev/null @@ -1,15 +0,0 @@ -# 5. Using Anchors for List Items -# You can use anchors for list items too. -features: &FEATURES - - wifi - - bluetooth - -phone1: - name: PhoneA - features: *FEATURES - -phone2: - name: PhoneB - features: - - camera - - *FEATURES diff --git a/libs/dyn/yamlloader/testdata/anchor_06.yml b/libs/dyn/yamlloader/testdata/anchor_06.yml deleted file mode 100644 index f27650fad47..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_06.yml +++ /dev/null @@ -1,5 +0,0 @@ -# 6. String Anchors -commonMessage: &msg "Hello, World!" - -greeting1: *msg -greeting2: *msg diff --git a/libs/dyn/yamlloader/testdata/anchor_07.yml b/libs/dyn/yamlloader/testdata/anchor_07.yml deleted file mode 100644 index b8f32214568..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_07.yml +++ /dev/null @@ -1,10 +0,0 @@ -# 7. Number Anchors -defaultAge: &age 25 - -person1: - name: Alice - age: *age - -person2: - name: Bob - age: *age diff --git a/libs/dyn/yamlloader/testdata/anchor_08.yml b/libs/dyn/yamlloader/testdata/anchor_08.yml deleted file mode 100644 index 8b01e21f14b..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_08.yml +++ /dev/null @@ -1,10 +0,0 @@ -# 8. Boolean Anchors -isActive: &active true - -user1: - username: user1 - active: *active - -user2: - username: user2 - active: *active diff --git a/libs/dyn/yamlloader/testdata/anchor_09.yml b/libs/dyn/yamlloader/testdata/anchor_09.yml deleted file mode 100644 index 8005bf445ef..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_09.yml +++ /dev/null @@ -1,2 +0,0 @@ -anchortest: &x - child: *x diff --git a/libs/dyn/yamlloader/testdata/anchor_10.yml b/libs/dyn/yamlloader/testdata/anchor_10.yml deleted file mode 100644 index 013832ab3ac..00000000000 --- a/libs/dyn/yamlloader/testdata/anchor_10.yml +++ /dev/null @@ -1,5 +0,0 @@ -leaf: &leaf 1 -shared: &shared - value: *leaf -use1: *shared -use2: *shared diff --git a/libs/dyn/yamlloader/testdata/empty.yml b/libs/dyn/yamlloader/testdata/empty.yml deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/libs/dyn/yamlloader/testdata/error_01.yml b/libs/dyn/yamlloader/testdata/error_01.yml deleted file mode 100644 index 11b96486e32..00000000000 --- a/libs/dyn/yamlloader/testdata/error_01.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Use string anchor to extend a mapping. -str: &str "Hello world!" - -map: - <<: *str - key: value diff --git a/libs/dyn/yamlloader/testdata/error_02.yml b/libs/dyn/yamlloader/testdata/error_02.yml deleted file mode 100644 index 5e9c6276d09..00000000000 --- a/libs/dyn/yamlloader/testdata/error_02.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Use string anchor inside sequence to extend a mapping. -str: &str "Hello world!" - -map: - <<: [*str] - key: value diff --git a/libs/dyn/yamlloader/testdata/error_03.yml b/libs/dyn/yamlloader/testdata/error_03.yml deleted file mode 100644 index c4d3c397dd9..00000000000 --- a/libs/dyn/yamlloader/testdata/error_03.yml +++ /dev/null @@ -1,4 +0,0 @@ -# Extend a mapping with a literal string. -map: - <<: value - key: value diff --git a/libs/dyn/yamlloader/testdata/mix_01.yml b/libs/dyn/yamlloader/testdata/mix_01.yml deleted file mode 100644 index 98706bf6f01..00000000000 --- a/libs/dyn/yamlloader/testdata/mix_01.yml +++ /dev/null @@ -1,90 +0,0 @@ -# Scalars - -# Strings can be unquoted -name: John Doe - -# Strings can be single quoted -single_quoted_string: 'This is a single quoted string' - -# Strings can be double quoted (allows for escape sequences) -double_quoted_string: "This is a double quoted string with an escaped newline \n and tab \t." - -# Multiline string with folded style (newlines become spaces) -folded_style: > - This is a very - long string that - spans several lines. - -# Multiline string with literal block style (newlines are preserved) -literal_style: | - Line 1 - Line 2 - Line 3 - -# Integers -age: 30 - -# Floating point -price: 19.99 - -# Boolean values -is_student: true -is_employed: false - -# Null value -middle_name: ~ # null can also be represented with 'null' - -# Timestamp -timestamp: 2023-01-01T12:00:00Z - -# Sequences (lists) - -# Inline list -fruits: [apple, banana, cherry] - -# Block style list -books: - - 'Moby Dick' - - '1984' - - 'Pride and Prejudice' - -# Mappings (dictionaries) - -# Inline dictionary -address: {street: '123 Main St', city: 'Anytown', zip: '12345'} - -# Block style dictionary -employee: - first_name: Jane - last_name: Smith - age: 40 - -# Edge cases and advanced features - -# Strings that look like other types must be quoted -looks_like_number: '12345' -looks_like_boolean: 'yes' -looks_like_null: 'null' -looks_like_timestamp: '2023-01-01T12:00:00Z' - -# Using anchors and aliases to reuse properties -base_address: &base - street: '456 Elm St' - city: 'Sometown' - -office_address: - <<: *base # Merge the base address into this mapping - suite: 500 - -# Nested structures -users: - - name: Alice - age: 28 - interests: - - reading - - cycling - - name: Bob - age: 35 - interests: - - cooking - - hiking diff --git a/libs/dyn/yamlloader/testdata/mix_02.yml b/libs/dyn/yamlloader/testdata/mix_02.yml deleted file mode 100644 index 757d626dbfe..00000000000 --- a/libs/dyn/yamlloader/testdata/mix_02.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Scalars with special characters - -# Commas and square brackets in strings should be enclosed in quotes -special_chars: "[This, string] has, special chars." - -# Strings starting with reserved indicators must be quoted -reserved_string: "@not_a_directive" -colon_string: "this: looks like a mapping, but it's not" - -# Explicit data type declaration -explicit_string: !!str 12345 - -# Sequences with nested mappings and lists - -teams: - - - name: Team A - members: - - Alice - - Bob - tasks: - - task1: - status: incomplete - due_date: 2023-01-15 - - task2: - status: complete - -# Complex mapping keys -? | - Multi-line key - which is unusual but valid -: multi-line key's value - -"complex key: with colon": complex key's value - -# Set (a mapping with null values) -set_example: - item1: ~ - item2: ~ - -# Merge multiple mappings (with override) -base_colors: &colors - red: "#FF0000" - blue: "#0000FF" - green: "#00FF00" - -theme: - <<: *colors - blue: "#001122" # Overriding the blue color from base_colors diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.1.yml b/libs/dyn/yamlloader/testdata/spec_example_2.1.yml deleted file mode 100644 index c9e26274c09..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.1.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Example 2.1 Sequence of Scalars (ball players) - -- Mark McGwire -- Sammy Sosa -- Ken Griffey diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.10.yml b/libs/dyn/yamlloader/testdata/spec_example_2.10.yml deleted file mode 100644 index a3459dedfc8..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.10.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.10 Node for “Sammy Sosa” appears twice in this document - ---- -hr: -- Mark McGwire -# Following node labeled SS -- &SS Sammy Sosa -rbi: -- *SS # Subsequent occurrence -- Ken Griffey diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.11.yml b/libs/dyn/yamlloader/testdata/spec_example_2.11.yml deleted file mode 100644 index e3c5c115e02..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.11.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.11 Mapping between Sequences - -? - Detroit Tigers - - Chicago cubs -: - 2001-07-23 - -? [ New York Yankees, - Atlanta Braves ] -: [ 2001-07-02, 2001-08-12, - 2001-08-14 ] diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.12.yml b/libs/dyn/yamlloader/testdata/spec_example_2.12.yml deleted file mode 100644 index eb4a526f65b..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.12.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.12 Compact Nested Mapping - ---- -# Products purchased -- item : Super Hoop - quantity: 1 -- item : Basketball - quantity: 4 -- item : Big Shoes - quantity: 1 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.13.yml b/libs/dyn/yamlloader/testdata/spec_example_2.13.yml deleted file mode 100644 index e55abff107e..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.13.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Example 2.13 In literals, newlines are preserved - -# ASCII Art ---- | - \//||\/|| - // || ||__ diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.14.yml b/libs/dyn/yamlloader/testdata/spec_example_2.14.yml deleted file mode 100644 index 439fca30cb1..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.14.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Example 2.14 In the folded scalars, newlines become spaces - ---- > - Mark McGwire's - year was crippled - by a knee injury. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.15.yml b/libs/dyn/yamlloader/testdata/spec_example_2.15.yml deleted file mode 100644 index 266e7ce4711..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.15.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.15 Folded newlines are preserved for “more indented” and blank lines - ---- > - Sammy Sosa completed another - fine season with great stats. - - 63 Home Runs - 0.288 Batting Average - - What a year! diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.16.yml b/libs/dyn/yamlloader/testdata/spec_example_2.16.yml deleted file mode 100644 index 6db6b0878f8..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.16.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Example 2.16 Indentation determines scope - -name: Mark McGwire -accomplishment: > - Mark set a major league - home run record in 1998. -stats: | - 65 Home Runs - 0.278 Batting Average diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.17.yml b/libs/dyn/yamlloader/testdata/spec_example_2.17.yml deleted file mode 100644 index af0777abab5..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.17.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Example 2.17 Quoted Scalars - -unicode: "Sosa did fine.\u263A" -control: "\b1998\t1999\t2000\n" -hex esc: "\x0d\x0a is \r\n" - -single: '"Howdy!" he cried.' -quoted: ' # Not a ''comment''.' -tie-fighter: '|\-*-/|' diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.18.yml b/libs/dyn/yamlloader/testdata/spec_example_2.18.yml deleted file mode 100644 index 741bcd8c0c0..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.18.yml +++ /dev/null @@ -1,8 +0,0 @@ -# Example 2.18 Multi-line Flow Scalars - -plain: - This unquoted scalar - spans many lines. - -quoted: "So does this - quoted scalar.\n" diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.19.yml b/libs/dyn/yamlloader/testdata/spec_example_2.19.yml deleted file mode 100644 index 6ed95e092aa..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.19.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Example 2.19 Integers - -canonical: 12345 -decimal: +12345 -octal: 0o14 -hexadecimal: 0xC - -# Note: this example is not part of the spec but added for completeness. -# -# Octal numbers: -# - YAML 1.1: prefix is "0" -# - YAML 1.2: prefix is "0o" -# The "gopkg.in/yaml.v3" package accepts both for backwards compat. -# We accept only the YAML 1.2 prefix "0o". -octal11: 012345 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.2.yml b/libs/dyn/yamlloader/testdata/spec_example_2.2.yml deleted file mode 100644 index 29c16105e3b..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.2.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Example 2.2 Mapping Scalars to Scalars (player statistics) - -hr: 65 # Home runs -avg: 0.278 # Batting average -rbi: 147 # Runs Batted In diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.20.yml b/libs/dyn/yamlloader/testdata/spec_example_2.20.yml deleted file mode 100644 index 77a79a0c3e0..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.20.yml +++ /dev/null @@ -1,7 +0,0 @@ -# Example 2.20 Floating Point - -canonical: 1.23015e+3 -exponential: 12.3015e+02 -fixed: 1230.15 -negative infinity: -.inf -not a number: .nan diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.21.yml b/libs/dyn/yamlloader/testdata/spec_example_2.21.yml deleted file mode 100644 index cdb423c566e..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.21.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Example 2.21 Miscellaneous - -null: -booleans: [ true, false ] -string: '012345' diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.22.yml b/libs/dyn/yamlloader/testdata/spec_example_2.22.yml deleted file mode 100644 index bef2addfa07..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.22.yml +++ /dev/null @@ -1,6 +0,0 @@ -# Example 2.22 Timestamps - -canonical: 2001-12-15T02:59:43.1Z -iso8601: 2001-12-14t21:59:43.10-05:00 -spaced: 2001-12-14 21:59:43.10 -5 -date: 2002-12-14 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.23.yml b/libs/dyn/yamlloader/testdata/spec_example_2.23.yml deleted file mode 100644 index 56e9898e21b..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.23.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Example 2.23 Various Explicit Tags - ---- -not-date: !!str 2002-04-28 - -picture: !!binary | - R0lGODlhDAAMAIQAAP//9/X - 17unp5WZmZgAAAOfn515eXv - Pz7Y6OjuDg4J+fn5OTk6enp - 56enmleECcgggoBADs= - -application specific tag: !something | - The semantics of the tag - above may be different for - different documents. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.24.yml b/libs/dyn/yamlloader/testdata/spec_example_2.24.yml deleted file mode 100644 index f7c11d0fb11..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.24.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Example 2.24 Global Tags - -%TAG ! tag:clarkevans.com,2002: ---- !shape - # Use the ! handle for presenting - # tag:clarkevans.com,2002:circle -- !circle - center: &ORIGIN {x: 73, y: 129} - radius: 7 -- !line - start: *ORIGIN - finish: { x: 89, y: 102 } -- !label - start: *ORIGIN - color: 0xFFEEBB - text: Pretty vector drawing. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.25.yml b/libs/dyn/yamlloader/testdata/spec_example_2.25.yml deleted file mode 100644 index 73bac862755..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.25.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Example 2.25 Unordered Sets - -# Sets are represented as a -# Mapping where each key is -# associated with a null value ---- !!set -? Mark McGwire -? Sammy Sosa -? Ken Griffey diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.26.yml b/libs/dyn/yamlloader/testdata/spec_example_2.26.yml deleted file mode 100644 index 00863a6b854..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.26.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Example 2.26 Ordered Mappings - -# Ordered maps are represented as -# A sequence of mappings, with -# each mapping having one key ---- !!omap -- Mark McGwire: 65 -- Sammy Sosa: 63 -- Ken Griffey: 58 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.27.yml b/libs/dyn/yamlloader/testdata/spec_example_2.27.yml deleted file mode 100644 index fc9b460c299..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.27.yml +++ /dev/null @@ -1,31 +0,0 @@ -# Example 2.27 Invoice - ---- ! -invoice: 34843 -date : 2001-01-23 -bill-to: &id001 - given : Chris - family : Dumars - address: - lines: | - 458 Walkman Dr. - Suite #292 - city : Royal Oak - state : MI - postal : 48046 -ship-to: *id001 -product: -- sku : BL394D - quantity : 4 - description : Basketball - price : 450.00 -- sku : BL4438H - quantity : 1 - description : Super Hoop - price : 2392.00 -tax : 251.42 -total: 4443.52 -comments: - Late afternoon is best. - Backup contact is Nancy - Billsmer @ 338-4338. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.28.yml b/libs/dyn/yamlloader/testdata/spec_example_2.28.yml deleted file mode 100644 index 35369472967..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.28.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Example 2.28 Log File - ---- -Time: 2001-11-23 15:01:42 -5 -User: ed -Warning: - This is an error message - for the log file ---- -Time: 2001-11-23 15:02:31 -5 -User: ed -Warning: - A slightly different error - message. ---- -Date: 2001-11-23 15:03:17 -5 -User: ed -Fatal: - Unknown variable "bar" -Stack: -- file: TopClass.py - line: 23 - code: | - x = MoreObject("345\n") -- file: MoreClass.py - line: 58 - code: |- - foo = bar diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.3.yml b/libs/dyn/yamlloader/testdata/spec_example_2.3.yml deleted file mode 100644 index 70cbe07d391..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.3.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.3 Mapping Scalars to Sequences (ball clubs in each league) - -american: -- Boston Red Sox -- Detroit Tigers -- New York Yankees -national: -- New York Mets -- Chicago Cubs -- Atlanta Braves diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.4.yml b/libs/dyn/yamlloader/testdata/spec_example_2.4.yml deleted file mode 100644 index cce28625f56..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.4.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.4 Sequence of Mappings (players’ statistics) - -- - name: Mark McGwire - hr: 65 - avg: 0.278 -- - name: Sammy Sosa - hr: 63 - avg: 0.288 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.5.yml b/libs/dyn/yamlloader/testdata/spec_example_2.5.yml deleted file mode 100644 index a585faee790..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.5.yml +++ /dev/null @@ -1,5 +0,0 @@ -# Example 2.5 Sequence of Sequences - -- [name , hr, avg ] -- [Mark McGwire, 65, 0.278] -- [Sammy Sosa , 63, 0.288] diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.6.yml b/libs/dyn/yamlloader/testdata/spec_example_2.6.yml deleted file mode 100644 index cc137e5d56d..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.6.yml +++ /dev/null @@ -1,7 +0,0 @@ -# Example 2.6 Mapping of Mappings - -Mark McGwire: {hr: 65, avg: 0.278} -Sammy Sosa: { - hr: 63, - avg: 0.288, - } diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.7.yml b/libs/dyn/yamlloader/testdata/spec_example_2.7.yml deleted file mode 100644 index 35c2541d6cc..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.7.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Example 2.7 Two Documents in a Stream (each with a leading comment) - -# Ranking of 1998 home runs ---- -- Mark McGwire -- Sammy Sosa -- Ken Griffey - -# Team ranking ---- -- Chicago Cubs -- St Louis Cardinals diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.8.yml b/libs/dyn/yamlloader/testdata/spec_example_2.8.yml deleted file mode 100644 index ae6e8bf279f..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.8.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Example 2.8 Play by Play Feed from a Game - ---- -time: 20:03:20 -player: Sammy Sosa -action: strike (miss) -... ---- -time: 20:03:47 -player: Sammy Sosa -action: grand slam -... diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.9.yml b/libs/dyn/yamlloader/testdata/spec_example_2.9.yml deleted file mode 100644 index 75217b25ec9..00000000000 --- a/libs/dyn/yamlloader/testdata/spec_example_2.9.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Example 2.9 Single Document with Two Comments - ---- -hr: # 1998 hr ranking -- Mark McGwire -- Sammy Sosa -# 1998 rbi ranking -rbi: -- Sammy Sosa -- Ken Griffey diff --git a/libs/dyn/yamlloader/yaml.go b/libs/dyn/yamlloader/yaml.go deleted file mode 100644 index 29570747ab1..00000000000 --- a/libs/dyn/yamlloader/yaml.go +++ /dev/null @@ -1,23 +0,0 @@ -package yamlloader - -import ( - "errors" - "io" - - "github.com/databricks/cli/libs/dyn" - "go.yaml.in/yaml/v3" -) - -func LoadYAML(path string, r io.Reader) (dyn.Value, error) { - var node yaml.Node - dec := yaml.NewDecoder(r) - err := dec.Decode(&node) - if err != nil { - if errors.Is(err, io.EOF) { - return dyn.NilValue, nil - } - return dyn.InvalidValue, err - } - - return newLoader(path).load(&node) -} diff --git a/libs/dyn/yamlloader/yaml_anchor_test.go b/libs/dyn/yamlloader/yaml_anchor_test.go deleted file mode 100644 index 00f4576a3e2..00000000000 --- a/libs/dyn/yamlloader/yaml_anchor_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package yamlloader_test - -import ( - "bytes" - "os" - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestYAMLAnchor01(t *testing.T) { - file := "testdata/anchor_01.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - assert.True(t, self.Get("defaults").IsAnchor()) - assert.False(t, self.Get("shirt1").IsAnchor()) - assert.False(t, self.Get("shirt2").IsAnchor()) - - pattern := self.Get("shirt1").Get("pattern") - assert.Equal(t, "striped", pattern.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 8, Column: 12}, pattern.Location()) -} - -func TestYAMLAnchor02(t *testing.T) { - file := "testdata/anchor_02.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - color := self.Get("shirt").Get("color") - assert.Equal(t, "red", color.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 4, Column: 10}, color.Location()) - - primary := self.Get("shirt").Get("primary") - assert.Equal(t, "cotton", primary.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 8, Column: 12}, primary.Location()) - - pattern := self.Get("shirt").Get("pattern") - assert.Equal(t, "striped", pattern.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 13, Column: 12}, pattern.Location()) -} - -func TestYAMLAnchor03(t *testing.T) { - file := "testdata/anchor_03.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - // Assert the override took place. - blue := self.Get("shirt").Get("color") - assert.Equal(t, "blue", blue.AsAny()) - assert.Equal(t, file, blue.Location().File) - assert.Equal(t, 10, blue.Location().Line) - assert.Equal(t, 10, blue.Location().Column) -} - -func TestYAMLAnchor04(t *testing.T) { - file := "testdata/anchor_04.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - p1 := self.Get("person1").Get("address").Get("city") - assert.Equal(t, "San Francisco", p1.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 4, Column: 9}, p1.Location()) - - p2 := self.Get("person2").Get("address").Get("city") - assert.Equal(t, "Los Angeles", p2.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 16, Column: 11}, p2.Location()) -} - -func TestYAMLAnchor05(t *testing.T) { - file := "testdata/anchor_05.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - features := self.Get("phone1").Get("features") - assert.Equal(t, "wifi", features.Index(0).AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 4, Column: 5}, features.Index(0).Location()) - assert.Equal(t, "bluetooth", features.Index(1).AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 5, Column: 5}, features.Index(1).Location()) -} - -func TestYAMLAnchor06(t *testing.T) { - file := "testdata/anchor_06.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - greeting := self.Get("greeting1") - assert.Equal(t, "Hello, World!", greeting.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 2, Column: 16}, greeting.Location()) -} - -func TestYAMLAnchor07(t *testing.T) { - file := "testdata/anchor_07.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - name := self.Get("person1").Get("name") - assert.Equal(t, "Alice", name.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 5, Column: 9}, name.Location()) - - age := self.Get("person1").Get("age") - assert.Equal(t, 25, age.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 2, Column: 13}, age.Location()) -} - -func TestYAMLAnchor08(t *testing.T) { - file := "testdata/anchor_08.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - username := self.Get("user1").Get("username") - assert.Equal(t, "user1", username.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 5, Column: 13}, username.Location()) - - active := self.Get("user1").Get("active") - assert.Equal(t, true, active.AsAny()) - assert.Equal(t, diag.Location{File: file, Line: 2, Column: 11}, active.Location()) -} - -func TestYAMLAnchor09(t *testing.T) { - file := "testdata/anchor_09.yml" - input, err := os.ReadFile(file) - require.NoError(t, err) - - _, err = yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - assert.ErrorContains(t, err, `cyclic reference to anchor "x"`) -} - -func TestYAMLAnchor10(t *testing.T) { - file := "testdata/anchor_10.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - assert.Equal(t, 1, self.Get("use1").Get("value").AsAny()) - assert.Equal(t, 1, self.Get("use2").Get("value").AsAny()) -} diff --git a/libs/dyn/yamlloader/yaml_error_test.go b/libs/dyn/yamlloader/yaml_error_test.go deleted file mode 100644 index a18f9599b05..00000000000 --- a/libs/dyn/yamlloader/yaml_error_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package yamlloader_test - -import ( - "bytes" - "os" - "testing" - - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.yaml.in/yaml/v3" -) - -func TestYAMLErrorMapMerge(t *testing.T) { - for _, file := range []string{ - "testdata/error_01.yml", - "testdata/error_02.yml", - "testdata/error_03.yml", - } { - input, err := os.ReadFile(file) - require.NoError(t, err) - - t.Run(file, func(t *testing.T) { - t.Run("reference", func(t *testing.T) { - var ref any - err = yaml.Unmarshal(input, &ref) - assert.ErrorContains(t, err, "map merge requires map or sequence of maps as the value") - }) - - t.Run("self", func(t *testing.T) { - _, err := yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - assert.ErrorContains(t, err, "map merge requires map or sequence of maps as the value") - }) - }) - } -} diff --git a/libs/dyn/yamlloader/yaml_mix_test.go b/libs/dyn/yamlloader/yaml_mix_test.go deleted file mode 100644 index 307b93dbf3a..00000000000 --- a/libs/dyn/yamlloader/yaml_mix_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package yamlloader_test - -import ( - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/stretchr/testify/assert" -) - -func TestYAMLMix01(t *testing.T) { - file := "testdata/mix_01.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - assert.True(t, self.Get("base_address").IsAnchor()) - assert.False(t, self.Get("office_address").IsAnchor()) -} - -func TestYAMLMix02(t *testing.T) { - file := "testdata/mix_02.yml" - self := loadYAML(t, file) - assert.NotEqual(t, dyn.NilValue, self) - - assert.True(t, self.Get("base_colors").IsAnchor()) - assert.False(t, self.Get("theme").IsAnchor()) -} diff --git a/libs/dyn/yamlloader/yaml_spec_test.go b/libs/dyn/yamlloader/yaml_spec_test.go deleted file mode 100644 index 28ad2e7480c..00000000000 --- a/libs/dyn/yamlloader/yaml_spec_test.go +++ /dev/null @@ -1,824 +0,0 @@ -package yamlloader_test - -import ( - "bytes" - "math" - "os" - "testing" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynassert" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const NL = "\n" - -func loadExample(t *testing.T, file string) dyn.Value { - input, err := os.ReadFile(file) - require.NoError(t, err) - self, err := yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - require.NoError(t, err) - return self -} - -func TestYAMLSpecExample_2_1(t *testing.T) { - file := "testdata/spec_example_2.1.yml" - self := loadExample(t, file) - - assert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Mark McGwire", []diag.Location{{File: file, Line: 3, Column: 3}}), - dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 4, Column: 3}}), - dyn.NewValue("Ken Griffey", []diag.Location{{File: file, Line: 5, Column: 3}}), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_2(t *testing.T) { - file := "testdata/spec_example_2.2.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue(65, []diag.Location{{File: file, Line: 3, Column: 6}}), - "avg": dyn.NewValue(0.278, []diag.Location{{File: file, Line: 4, Column: 6}}), - "rbi": dyn.NewValue(147, []diag.Location{{File: file, Line: 5, Column: 6}}), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_3(t *testing.T) { - file := "testdata/spec_example_2.3.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "american": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Boston Red Sox", []diag.Location{{File: file, Line: 4, Column: 3}}), - dyn.NewValue("Detroit Tigers", []diag.Location{{File: file, Line: 5, Column: 3}}), - dyn.NewValue("New York Yankees", []diag.Location{{File: file, Line: 6, Column: 3}}), - }, - []diag.Location{{File: file, Line: 4, Column: 1}}, - ), - "national": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("New York Mets", []diag.Location{{File: file, Line: 8, Column: 3}}), - dyn.NewValue("Chicago Cubs", []diag.Location{{File: file, Line: 9, Column: 3}}), - dyn.NewValue("Atlanta Braves", []diag.Location{{File: file, Line: 10, Column: 3}}), - }, - []diag.Location{{File: file, Line: 8, Column: 1}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_4(t *testing.T) { - file := "testdata/spec_example_2.4.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "name": dyn.NewValue("Mark McGwire", []diag.Location{{File: file, Line: 4, Column: 9}}), - "hr": dyn.NewValue(65, []diag.Location{{File: file, Line: 5, Column: 9}}), - "avg": dyn.NewValue(0.278, []diag.Location{{File: file, Line: 6, Column: 9}}), - }, - []diag.Location{{File: file, Line: 4, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "name": dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 8, Column: 9}}), - "hr": dyn.NewValue(63, []diag.Location{{File: file, Line: 9, Column: 9}}), - "avg": dyn.NewValue(0.288, []diag.Location{{File: file, Line: 10, Column: 9}}), - }, - []diag.Location{{File: file, Line: 8, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_5(t *testing.T) { - file := "testdata/spec_example_2.5.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - []dyn.Value{ - dyn.NewValue("name", []diag.Location{{File: file, Line: 3, Column: 4}}), - dyn.NewValue("hr", []diag.Location{{File: file, Line: 3, Column: 18}}), - dyn.NewValue("avg", []diag.Location{{File: file, Line: 3, Column: 22}}), - }, - []diag.Location{{File: file, Line: 3, Column: 3}}, - ), - dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Mark McGwire", []diag.Location{{File: file, Line: 4, Column: 4}}), - dyn.NewValue(65, []diag.Location{{File: file, Line: 4, Column: 18}}), - dyn.NewValue(0.278, []diag.Location{{File: file, Line: 4, Column: 22}}), - }, - []diag.Location{{File: file, Line: 4, Column: 3}}, - ), - dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 5, Column: 4}}), - dyn.NewValue(63, []diag.Location{{File: file, Line: 5, Column: 18}}), - dyn.NewValue(0.288, []diag.Location{{File: file, Line: 5, Column: 22}}), - }, - []diag.Location{{File: file, Line: 5, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_6(t *testing.T) { - file := "testdata/spec_example_2.6.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "Mark McGwire": dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue(65, []diag.Location{{File: file, Line: 3, Column: 20}}), - "avg": dyn.NewValue(0.278, []diag.Location{{File: file, Line: 3, Column: 29}}), - }, - []diag.Location{{File: file, Line: 3, Column: 15}}, - ), - "Sammy Sosa": dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue(63, []diag.Location{{File: file, Line: 5, Column: 9}}), - "avg": dyn.NewValue(0.288, []diag.Location{{File: file, Line: 6, Column: 10}}), - }, - []diag.Location{{File: file, Line: 4, Column: 13}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_7(t *testing.T) { - file := "testdata/spec_example_2.7.yml" - self := loadExample(t, file) - - // Note: we do not support multiple documents in a single YAML file. - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - "Mark McGwire", - []diag.Location{{File: file, Line: 5, Column: 3}}, - ), - dyn.NewValue( - "Sammy Sosa", - []diag.Location{{File: file, Line: 6, Column: 3}}, - ), - dyn.NewValue( - "Ken Griffey", - []diag.Location{{File: file, Line: 7, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 5, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_8(t *testing.T) { - file := "testdata/spec_example_2.8.yml" - self := loadExample(t, file) - - // Note: we do not support multiple documents in a single YAML file. - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "time": dyn.NewValue("20:03:20", []diag.Location{{File: file, Line: 4, Column: 7}}), - "player": dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 5, Column: 9}}), - "action": dyn.NewValue("strike (miss)", []diag.Location{{File: file, Line: 6, Column: 9}}), - }, - []diag.Location{{File: file, Line: 4, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_9(t *testing.T) { - file := "testdata/spec_example_2.9.yml" - self := loadExample(t, file) - - // Note: we do not support multiple documents in a single YAML file. - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Mark McGwire", []diag.Location{{File: file, Line: 5, Column: 3}}), - dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 6, Column: 3}}), - }, - []diag.Location{{File: file, Line: 5, Column: 1}}, - ), - "rbi": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 9, Column: 3}}), - dyn.NewValue("Ken Griffey", []diag.Location{{File: file, Line: 10, Column: 3}}), - }, - []diag.Location{{File: file, Line: 9, Column: 1}}, - ), - }, - []diag.Location{{File: file, Line: 4, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_10(t *testing.T) { - file := "testdata/spec_example_2.10.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "hr": dyn.NewValue( - []dyn.Value{ - dyn.NewValue("Mark McGwire", []diag.Location{{File: file, Line: 5, Column: 3}}), - dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 7, Column: 3}}), - }, - []diag.Location{{File: file, Line: 5, Column: 1}}, - ), - "rbi": dyn.NewValue( - []dyn.Value{ - // The location for an anchored value refers to the anchor, not the reference. - // This is the same location as the anchor that appears in the "hr" mapping. - dyn.NewValue("Sammy Sosa", []diag.Location{{File: file, Line: 7, Column: 3}}), - dyn.NewValue("Ken Griffey", []diag.Location{{File: file, Line: 10, Column: 3}}), - }, - []diag.Location{{File: file, Line: 9, Column: 1}}, - ), - }, - []diag.Location{{File: file, Line: 4, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_11(t *testing.T) { - file := "testdata/spec_example_2.11.yml" - input, err := os.ReadFile(file) - require.NoError(t, err) - - // Note: non-string mapping keys are not supported by "go.yaml.in/yaml/v3". - _, err = yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - assert.ErrorContains(t, err, `: key is not a scalar`) -} - -func TestYAMLSpecExample_2_12(t *testing.T) { - file := "testdata/spec_example_2.12.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "item": dyn.NewValue("Super Hoop", []diag.Location{{File: file, Line: 5, Column: 13}}), - "quantity": dyn.NewValue(1, []diag.Location{{File: file, Line: 6, Column: 13}}), - }, - []diag.Location{{File: file, Line: 5, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "item": dyn.NewValue("Basketball", []diag.Location{{File: file, Line: 7, Column: 13}}), - "quantity": dyn.NewValue(4, []diag.Location{{File: file, Line: 8, Column: 13}}), - }, - []diag.Location{{File: file, Line: 7, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "item": dyn.NewValue("Big Shoes", []diag.Location{{File: file, Line: 9, Column: 13}}), - "quantity": dyn.NewValue(1, []diag.Location{{File: file, Line: 10, Column: 13}}), - }, - []diag.Location{{File: file, Line: 9, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 5, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_13(t *testing.T) { - file := "testdata/spec_example_2.13.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - ``+ - `\//||\/||`+NL+ - "// || ||__"+NL, - []diag.Location{{File: file, Line: 4, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_14(t *testing.T) { - file := "testdata/spec_example_2.14.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - `Mark McGwire's year was crippled by a knee injury.`+NL, - []diag.Location{{File: file, Line: 3, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_15(t *testing.T) { - file := "testdata/spec_example_2.15.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - ``+ - `Sammy Sosa completed another fine season with great stats.`+NL+ - NL+ - ` 63 Home Runs`+NL+ - ` 0.288 Batting Average`+NL+ - NL+ - `What a year!`+NL, - []diag.Location{{File: file, Line: 3, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_16(t *testing.T) { - file := "testdata/spec_example_2.16.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "name": dyn.NewValue( - "Mark McGwire", - []diag.Location{{File: file, Line: 3, Column: 7}}, - ), - "accomplishment": dyn.NewValue( - `Mark set a major league home run record in 1998.`+NL, - []diag.Location{{File: file, Line: 4, Column: 17}}, - ), - "stats": dyn.NewValue( - ``+ - `65 Home Runs`+NL+ - `0.278 Batting Average`+NL, - []diag.Location{{File: file, Line: 7, Column: 8}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_17(t *testing.T) { - file := "testdata/spec_example_2.17.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "unicode": dyn.NewValue( - `Sosa did fine.`+"\u263A", - []diag.Location{{File: file, Line: 3, Column: 10}}, - ), - "control": dyn.NewValue( - "\b1998\t1999\t2000\n", - []diag.Location{{File: file, Line: 4, Column: 10}}, - ), - "hex esc": dyn.NewValue( - "\x0d\x0a is \r\n", - []diag.Location{{File: file, Line: 5, Column: 10}}, - ), - "single": dyn.NewValue( - `"Howdy!" he cried.`, - []diag.Location{{File: file, Line: 7, Column: 9}}, - ), - "quoted": dyn.NewValue( - ` # Not a 'comment'.`, - []diag.Location{{File: file, Line: 8, Column: 9}}, - ), - "tie-fighter": dyn.NewValue( - `|\-*-/|`, - []diag.Location{{File: file, Line: 9, Column: 14}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_18(t *testing.T) { - file := "testdata/spec_example_2.18.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "plain": dyn.NewValue( - `This unquoted scalar spans many lines.`, - []diag.Location{{File: file, Line: 4, Column: 3}}, - ), - "quoted": dyn.NewValue( - `So does this quoted scalar.`+NL, - []diag.Location{{File: file, Line: 7, Column: 9}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_19(t *testing.T) { - file := "testdata/spec_example_2.19.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "canonical": dyn.NewValue( - 12345, - []diag.Location{{File: file, Line: 3, Column: 12}}, - ), - "decimal": dyn.NewValue( - 12345, - []diag.Location{{File: file, Line: 4, Column: 10}}, - ), - "octal": dyn.NewValue( - 12, - []diag.Location{{File: file, Line: 5, Column: 8}}, - ), - "hexadecimal": dyn.NewValue( - 12, - []diag.Location{{File: file, Line: 6, Column: 14}}, - ), - "octal11": dyn.NewValue( - 12345, - []diag.Location{{File: file, Line: 15, Column: 10}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_20(t *testing.T) { - file := "testdata/spec_example_2.20.yml" - self := loadExample(t, file) - - // Equality assertion doesn't work with NaNs. - // See https://github.com/stretchr/testify/issues/624. - // - // Remove the NaN entry. - self, _ = dyn.Walk(self, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if f, ok := v.AsFloat(); ok && math.IsNaN(f) { - return dyn.InvalidValue, dyn.ErrDrop - } - return v, nil - }) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "canonical": dyn.NewValue( - 1230.15, - []diag.Location{{File: file, Line: 3, Column: 12}}, - ), - "exponential": dyn.NewValue( - 1230.15, - []diag.Location{{File: file, Line: 4, Column: 14}}, - ), - "fixed": dyn.NewValue( - 1230.15, - []diag.Location{{File: file, Line: 5, Column: 8}}, - ), - "negative infinity": dyn.NewValue( - math.Inf(-1), - []diag.Location{{File: file, Line: 6, Column: 20}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_21(t *testing.T) { - file := "testdata/spec_example_2.21.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "null": dyn.NewValue( - nil, - []diag.Location{{File: file, Line: 3, Column: 6}}, - ), - "booleans": dyn.NewValue( - []dyn.Value{ - dyn.NewValue(true, []diag.Location{{File: file, Line: 4, Column: 13}}), - dyn.NewValue(false, []diag.Location{{File: file, Line: 4, Column: 19}}), - }, - []diag.Location{{File: file, Line: 4, Column: 11}}, - ), - "string": dyn.NewValue( - "012345", - []diag.Location{{File: file, Line: 5, Column: 9}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_22(t *testing.T) { - file := "testdata/spec_example_2.22.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "canonical": dyn.NewValue( - dyn.MustTime("2001-12-15T02:59:43.1Z"), - []diag.Location{{File: file, Line: 3, Column: 12}}, - ), - "iso8601": dyn.NewValue( - dyn.MustTime("2001-12-14t21:59:43.10-05:00"), - []diag.Location{{File: file, Line: 4, Column: 10}}, - ), - "spaced": dyn.NewValue( - // This is parsed as a string, not a timestamp, - // both by "go.yaml.in/yaml/v3" and by our implementation. - "2001-12-14 21:59:43.10 -5", - []diag.Location{{File: file, Line: 5, Column: 9}}, - ), - "date": dyn.NewValue( - dyn.MustTime("2002-12-14"), - []diag.Location{{File: file, Line: 6, Column: 7}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 1}}, - ), self) -} - -func TestYAMLSpecExample_2_23(t *testing.T) { - file := "testdata/spec_example_2.23.yml" - input, err := os.ReadFile(file) - require.NoError(t, err) - - // Note: the !!binary tag is not supported by us. - - _, err = yamlloader.LoadYAML(file, bytes.NewBuffer(input)) - assert.ErrorContains(t, err, `: unknown tag: !!binary`) -} - -func TestYAMLSpecExample_2_24(t *testing.T) { - file := "testdata/spec_example_2.24.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "center": dyn.NewValue( - map[string]dyn.Value{ - "x": dyn.NewValue(73, []diag.Location{{File: file, Line: 8, Column: 23}}), - "y": dyn.NewValue(129, []diag.Location{{File: file, Line: 8, Column: 30}}), - }, - []diag.Location{{File: file, Line: 8, Column: 11}}, - ), - "radius": dyn.NewValue(7, []diag.Location{{File: file, Line: 9, Column: 11}}), - }, - []diag.Location{{File: file, Line: 7, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "start": dyn.NewValue( - map[string]dyn.Value{ - "x": dyn.NewValue(73, []diag.Location{{File: file, Line: 8, Column: 23}}), - "y": dyn.NewValue(129, []diag.Location{{File: file, Line: 8, Column: 30}}), - }, - []diag.Location{{File: file, Line: 8, Column: 11}}, - ), - "finish": dyn.NewValue( - map[string]dyn.Value{ - "x": dyn.NewValue(89, []diag.Location{{File: file, Line: 12, Column: 16}}), - "y": dyn.NewValue(102, []diag.Location{{File: file, Line: 12, Column: 23}}), - }, - []diag.Location{{File: file, Line: 12, Column: 11}}, - ), - }, - []diag.Location{{File: file, Line: 10, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "start": dyn.NewValue( - map[string]dyn.Value{ - "x": dyn.NewValue(73, []diag.Location{{File: file, Line: 8, Column: 23}}), - "y": dyn.NewValue(129, []diag.Location{{File: file, Line: 8, Column: 30}}), - }, - []diag.Location{{File: file, Line: 8, Column: 11}}, - ), - "color": dyn.NewValue(16772795, []diag.Location{{File: file, Line: 15, Column: 10}}), - "text": dyn.NewValue("Pretty vector drawing.", []diag.Location{{File: file, Line: 16, Column: 9}}), - }, - []diag.Location{{File: file, Line: 13, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 4, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_25(t *testing.T) { - file := "testdata/spec_example_2.25.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "Mark McGwire": dyn.NewValue(nil, []diag.Location{{File: file, Line: 8, Column: 1}}), - "Sammy Sosa": dyn.NewValue(nil, []diag.Location{{File: file, Line: 9, Column: 1}}), - "Ken Griffey": dyn.NewValue(nil, []diag.Location{{File: file, Line: 10, Column: 1}}), - }, - []diag.Location{{File: file, Line: 6, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_26(t *testing.T) { - file := "testdata/spec_example_2.26.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "Mark McGwire": dyn.NewValue(65, []diag.Location{{File: file, Line: 7, Column: 17}}), - }, - []diag.Location{{File: file, Line: 7, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "Sammy Sosa": dyn.NewValue(63, []diag.Location{{File: file, Line: 8, Column: 15}}), - }, - []diag.Location{{File: file, Line: 8, Column: 3}}, - ), - dyn.NewValue( - map[string]dyn.Value{ - "Ken Griffey": dyn.NewValue(58, []diag.Location{{File: file, Line: 9, Column: 16}}), - }, - []diag.Location{{File: file, Line: 9, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 6, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_27(t *testing.T) { - file := "testdata/spec_example_2.27.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "invoice": dyn.NewValue( - 34843, - []diag.Location{{File: file, Line: 4, Column: 10}}, - ), - "date": dyn.NewValue( - dyn.MustTime("2001-01-23"), - []diag.Location{{File: file, Line: 5, Column: 10}}, - ), - "bill-to": dyn.NewValue( - map[string]dyn.Value{ - "given": dyn.NewValue( - "Chris", - []diag.Location{{File: file, Line: 7, Column: 12}}, - ), - "family": dyn.NewValue( - "Dumars", - []diag.Location{{File: file, Line: 8, Column: 12}}, - ), - "address": dyn.NewValue( - map[string]dyn.Value{ - "lines": dyn.NewValue( - "458 Walkman Dr.\nSuite #292\n", - []diag.Location{{File: file, Line: 10, Column: 12}}, - ), - "city": dyn.NewValue( - "Royal Oak", - []diag.Location{{File: file, Line: 13, Column: 15}}, - ), - "state": dyn.NewValue( - "MI", - []diag.Location{{File: file, Line: 14, Column: 15}}, - ), - "postal": dyn.NewValue( - 48046, - []diag.Location{{File: file, Line: 15, Column: 15}}, - ), - }, - []diag.Location{{File: file, Line: 10, Column: 5}}, - ), - }, - []diag.Location{{File: file, Line: 6, Column: 10}}, - ), - "ship-to": dyn.NewValue( - map[string]dyn.Value{ - "given": dyn.NewValue( - "Chris", - []diag.Location{{File: file, Line: 7, Column: 12}}, - ), - "family": dyn.NewValue( - "Dumars", - []diag.Location{{File: file, Line: 8, Column: 12}}, - ), - "address": dyn.NewValue( - map[string]dyn.Value{ - "lines": dyn.NewValue( - "458 Walkman Dr.\nSuite #292\n", - []diag.Location{{File: file, Line: 10, Column: 12}}, - ), - "city": dyn.NewValue( - "Royal Oak", - []diag.Location{{File: file, Line: 13, Column: 15}}, - ), - "state": dyn.NewValue( - "MI", - []diag.Location{{File: file, Line: 14, Column: 15}}, - ), - "postal": dyn.NewValue( - 48046, - []diag.Location{{File: file, Line: 15, Column: 15}}, - ), - }, - []diag.Location{{File: file, Line: 10, Column: 5}}, - ), - }, - []diag.Location{{File: file, Line: 6, Column: 10}}, - ), - "product": dyn.NewValue( - []dyn.Value{ - dyn.NewValue( - map[string]dyn.Value{ - "sku": dyn.NewValue( - "BL394D", - []diag.Location{{File: file, Line: 18, Column: 17}}, - ), - "quantity": dyn.NewValue( - 4, - []diag.Location{{File: file, Line: 19, Column: 17}}, - ), - "description": dyn.NewValue( - "Basketball", - []diag.Location{{File: file, Line: 20, Column: 17}}, - ), - "price": dyn.NewValue( - 450.0, - []diag.Location{{File: file, Line: 21, Column: 17}}, - ), - }, - []diag.Location{{File: file, Line: 18, Column: 3}}, - ), dyn.NewValue( - map[string]dyn.Value{ - "sku": dyn.NewValue( - "BL4438H", - []diag.Location{{File: file, Line: 22, Column: 17}}, - ), - "quantity": dyn.NewValue( - 1, - []diag.Location{{File: file, Line: 23, Column: 17}}, - ), - "description": dyn.NewValue( - "Super Hoop", - []diag.Location{{File: file, Line: 24, Column: 17}}, - ), - "price": dyn.NewValue( - 2392.0, - []diag.Location{{File: file, Line: 25, Column: 17}}, - ), - }, - []diag.Location{{File: file, Line: 22, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 18, Column: 1}}, - ), - "tax": dyn.NewValue( - 251.42, - []diag.Location{{File: file, Line: 26, Column: 8}}, - ), - "total": dyn.NewValue( - 4443.52, - []diag.Location{{File: file, Line: 27, Column: 8}}, - ), - "comments": dyn.NewValue( - "Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338.", - []diag.Location{{File: file, Line: 29, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 3, Column: 5}}, - ), self) -} - -func TestYAMLSpecExample_2_28(t *testing.T) { - file := "testdata/spec_example_2.28.yml" - self := loadExample(t, file) - - dynassert.Equal(t, dyn.NewValue( - map[string]dyn.Value{ - "Time": dyn.NewValue( - "2001-11-23 15:01:42 -5", - []diag.Location{{File: file, Line: 4, Column: 7}}, - ), - "User": dyn.NewValue( - "ed", - []diag.Location{{File: file, Line: 5, Column: 7}}, - ), - "Warning": dyn.NewValue( - "This is an error message for the log file", - []diag.Location{{File: file, Line: 7, Column: 3}}, - ), - }, - []diag.Location{{File: file, Line: 4, Column: 1}}, - ), self) -} diff --git a/libs/dyn/yamlloader/yaml_test.go b/libs/dyn/yamlloader/yaml_test.go deleted file mode 100644 index e21065e92c0..00000000000 --- a/libs/dyn/yamlloader/yaml_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package yamlloader_test - -import ( - "bytes" - "os" - "testing" - - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/yamlloader" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.yaml.in/yaml/v3" -) - -func loadYAML(t *testing.T, path string) dyn.Value { - input, err := os.ReadFile(path) - require.NoError(t, err) - - var ref any - err = yaml.Unmarshal(input, &ref) - require.NoError(t, err) - - self, err := yamlloader.LoadYAML(path, bytes.NewBuffer(input)) - require.NoError(t, err) - assert.NotNil(t, self) - - // Deep-equal the two values to ensure that the loader is producing - assert.EqualValues(t, ref, self.AsAny()) - return self -} - -func TestYAMLEmpty(t *testing.T) { - self := loadYAML(t, "testdata/empty.yml") - assert.Equal(t, dyn.NilValue, self) -} diff --git a/libs/structs/structvar/decode.go b/libs/structs/structvar/decode.go index bc6400eba76..9914e7e036b 100644 --- a/libs/structs/structvar/decode.go +++ b/libs/structs/structvar/decode.go @@ -120,7 +120,18 @@ func (sv *StructVar) Assign(path *structpath.PathNode, src View) (diag.Diagnosti return d.diags, nil } +// validate reports the first YAML error in the document, in the order loading it would +// find them, also in values that decoding drops. func validate(s source) error { + ys, ok := s.(*yamlSource) + if !ok { + return validateSource(s) + } + return ys.validateAll() +} + +// validateSource validates a source that is not YAML. +func validateSource(s source) error { switch s.kind() { case KindMap: pairs, err := s.pairs() @@ -128,7 +139,7 @@ func validate(s source) error { return err } for _, p := range pairs { - if err := validate(p.value); err != nil { + if err := validateSource(p.value); err != nil { return err } } @@ -138,7 +149,7 @@ func validate(s source) error { return err } for _, e := range elems { - if err := validate(e); err != nil { + if err := validateSource(e); err != nil { return err } } diff --git a/libs/structs/structvar/decode_dyn.go b/libs/structs/structvar/decode_dyn.go deleted file mode 100644 index 1aa687e8970..00000000000 --- a/libs/structs/structvar/decode_dyn.go +++ /dev/null @@ -1,104 +0,0 @@ -package structvar - -import ( - "fmt" - "reflect" - - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" -) - -// DecodeDyn decodes the configuration tree v into dst, a pointer to a typed -// configuration value, and returns its references and locations (those of every -// value in v, including the ones merges accumulated). -// -// Values are converted to the field types like [DecodeYAML] converts them; the -// diagnostics explain values that were dropped. -func DecodeDyn(v dyn.Value, dst any) (*StructVar, diag.Diagnostics, error) { - sv := &StructVar{Value: dst} - rv := reflect.ValueOf(dst).Elem() - if !v.IsValid() { - rv.SetZero() - return sv, nil, nil - } - var d decoder - n, ok, err := d.decode(rv, dynSource{v}, nil) - if err == nil && !ok { - err = fmt.Errorf("expected a %s, found a %s", kindOf(rv.Type()), dynSource{v}.kind()) - } - sv.Refs = n.refs(nil, nil) - sv.Locations = n.locations() - return sv, d.diags, err -} - -// dynSource is a [source] backed by a [dyn.Value]. -type dynSource struct { - v dyn.Value -} - -func (s dynSource) kind() Kind { - switch s.v.Kind() { - case dyn.KindMap: - return KindMap - case dyn.KindSequence: - return KindSequence - case dyn.KindString: - return KindString - case dyn.KindBool: - return KindBool - case dyn.KindInt: - return KindInt - case dyn.KindFloat: - return KindFloat - case dyn.KindTime: - return KindTime - case dyn.KindNil: - return KindNil - default: - return KindInvalid - } -} - -func (s dynSource) locations() []diag.Location { return s.v.Locations() } -func (s dynSource) anchor() bool { return s.v.IsAnchor() } - -func (s dynSource) scalar() any { - switch s.v.Kind() { - case dyn.KindTime: - return s.v.MustTime().String() - case dyn.KindInt: - // Like YAML values: int if it fits, int64 otherwise. - i, _ := s.v.AsInt() - return intValue(i) - case dyn.KindFloat: - // The decoder expects float64. - f, _ := s.v.AsFloat() - return f - default: - return s.v.AsAny() - } -} - -func (s dynSource) pairs() ([]sourcePair, error) { - m, ok := s.v.AsMap() - if !ok { - return nil, nil - } - out := make([]sourcePair, 0, m.Len()) - for _, p := range m.Pairs() { - out = append(out, sourcePair{key: p.Key.MustString(), keyLoc: p.Key.Location(), value: dynSource{p.Value}}) - } - return out, nil -} - -func (s dynSource) elems() ([]source, error) { - seq, ok := s.v.AsSequence() - if !ok { - return nil, nil - } - out := make([]source, len(seq)) - for i, e := range seq { - out[i] = dynSource{e} - } - return out, nil -} diff --git a/libs/structs/structvar/decode_yaml.go b/libs/structs/structvar/decode_yaml.go index 0b3e85234ee..ba75a48adc3 100644 --- a/libs/structs/structvar/decode_yaml.go +++ b/libs/structs/structvar/decode_yaml.go @@ -59,6 +59,20 @@ func ParseYAML(r io.Reader) (*yaml.Node, error) { return doc.Content[0], nil } +// ValidateYAMLNode reports the first YAML error in the document node (nil if it is empty) +// that decoding would report, also in values that decoding drops: a merge key with an +// invalid value, two merge keys in a mapping, or a key that is not a string. +func ValidateYAMLNode(file string, node *yaml.Node) error { + if node == nil { + return nil + } + src, err := newYAMLSource(file, node, nil) + if err != nil { + return err + } + return validate(src) +} + // aliasChain is an immutable list of alias nodes. type aliasChain struct { node *yaml.Node @@ -314,6 +328,94 @@ func (s *yamlSource) pairs() ([]sourcePair, error) { return out.pairs, nil } +// validateAll reports the first YAML error in the value: a key that is not a string, two +// merge keys in a mapping, or a merge key with a value that is not a mapping or a sequence +// of mappings. Unlike [yamlSource.pairs] it visits the values that a later duplicate key +// shadows. +func (s *yamlSource) validateAll() error { + switch s.k { + case KindSequence: + for _, n := range s.target.Content { + c, err := s.child(n, noPath) + if err != nil { + return err + } + if err := c.validateAll(); err != nil { + return err + } + } + case KindMap: + return s.validateMapping() + default: + } + return nil +} + +// noPath is the path of a value that is only validated. +func noPath() *structpath.PathNode { return nil } + +func (s *yamlSource) validateMapping() error { + node := s.target + loc := s.loc(node) + var merge *yaml.Node + for i := 0; i < len(node.Content); i += 2 { + key := node.Content[i] + val := node.Content[i+1] + + if key.Kind != yaml.ScalarNode { + return yamlErrorf(loc, "key is not a scalar") + } + switch st := key.ShortTag(); st { + case "!!str", "!!null": + case "!!merge": + if merge != nil { + return &LocationError{ + Loc: s.loc(key), + Summary: "duplicate YAML merge key ('<<') is not allowed; to merge multiple maps, use a sequence: '<<: [*anchor1, *anchor2]'", + } + } + merge = val + continue + default: + return yamlErrorf(loc, "invalid key tag: %v", st) + } + c, err := s.child(val, noPath) + if err != nil { + return err + } + if err := c.validateAll(); err != nil { + return err + } + } + if merge == nil { + return nil + } + + merr := yamlErrorf(s.loc(merge), "map merge requires map or sequence of maps as the value") + var mnodes []*yaml.Node + switch merge.Kind { + case yaml.SequenceNode: + mnodes = merge.Content + case yaml.AliasNode: + mnodes = []*yaml.Node{merge} + default: + return merr + } + for _, n := range mnodes { + c, err := s.child(n, noPath) + if err != nil { + return err + } + if c.k != KindMap { + return merr + } + if err := c.validateAll(); err != nil { + return err + } + } + return nil +} + // setPair sets the pair in pairs: an existing key keeps its position and key location. // pairSet is a list of pairs with unique keys, in the order the keys were first set. type pairSet struct { diff --git a/yamlfmt.yml b/yamlfmt.yml index 1b95edf2ade..f76d5d66271 100644 --- a/yamlfmt.yml +++ b/yamlfmt.yml @@ -1,7 +1,6 @@ exclude: - acceptance/selftest/bundleconfig # https://github.com/google/yamlfmt/issues/254 - acceptance/bundle/artifacts/nil_artifacts/databricks.yml # https://github.com/google/yamlfmt/issues/253 - - libs/dyn/yamlloader/testdata - libs/structs/structvar/testdata/yaml - bundle/direct/dresources/apitypes.yml # remove comments - acceptance/bundle/resources/permissions/jobs/update/databricks.yml # comment indentation is significant: it is uncommented into a top-level permissions: [] key