From 57aea946adca512db8a50c342b87a00391c0c487 Mon Sep 17 00:00:00 2001 From: Clifford Tawiah Date: Mon, 9 Nov 2026 13:44:47 -0500 Subject: [PATCH] feat(sync): add the structured document adapters Add the JSON, YAML, and TOML adapters. Each scalar has an exact byte span. Objects and arrays keep the layout of the text that they replace. Anchors, aliases, merge keys, block scalars, and arrays of tables are read-only. Co-Authored-By: Claude Fable 5.1 --- internal/sync/adapter/node.go | 78 ++- internal/sync/adapter/reference.go | 17 +- internal/sync/adapter/structured/json.go | 289 +++++++++++ internal/sync/adapter/structured/problem.go | 12 + .../adapter/structured/structured_test.go | 304 +++++++++++ .../adapter/structured/testdata/agents.yaml | 16 + .../sync/adapter/structured/testdata/mcp.json | 32 ++ .../structured/testdata/pyproject.toml | 20 + internal/sync/adapter/structured/toml.go | 490 ++++++++++++++++++ internal/sync/adapter/structured/toml_test.go | 218 ++++++++ internal/sync/adapter/structured/yaml.go | 75 +++ internal/sync/adapter/yamltree/yamltree.go | 52 +- 12 files changed, 1590 insertions(+), 13 deletions(-) create mode 100644 internal/sync/adapter/structured/json.go create mode 100644 internal/sync/adapter/structured/problem.go create mode 100644 internal/sync/adapter/structured/structured_test.go create mode 100644 internal/sync/adapter/structured/testdata/agents.yaml create mode 100644 internal/sync/adapter/structured/testdata/mcp.json create mode 100644 internal/sync/adapter/structured/testdata/pyproject.toml create mode 100644 internal/sync/adapter/structured/toml.go create mode 100644 internal/sync/adapter/structured/toml_test.go create mode 100644 internal/sync/adapter/structured/yaml.go diff --git a/internal/sync/adapter/node.go b/internal/sync/adapter/node.go index 24cd82812..1c05c4fb1 100644 --- a/internal/sync/adapter/node.go +++ b/internal/sync/adapter/node.go @@ -31,6 +31,38 @@ type Node struct { Encode func(any) ([]byte, error) } +// PositionalIDReason marks the id of a resource that has no name of its own, +// such as the second unnamed agent of a file. The id is the position of the +// resource, so a selection by it is as fragile as an index. +const PositionalIDReason = "the id is the position of a resource without a name" + +// MaxExactInteger is the largest integer that a float64 holds exactly. A +// number value is a float64, so a larger integer in a source stays read-only +// text instead of a rounded number. +const MaxExactInteger = int64(1) << 53 + +// ExactInteger reports whether a float64 holds an integer exactly. +func ExactInteger(integer int64) bool { + return integer >= -MaxExactInteger && integer <= MaxExactInteger +} + +// PositionalID returns an id node that marks a positional id. +func PositionalID(id string) *Node { + return &Node{Value: id, Capability: CapabilityReadOnly, Reason: PositionalIDReason} +} + +// IsPositionalID reports whether a resource has a positional id. +func IsPositionalID(resource *Node) bool { + if resource == nil || resource.Fields == nil { + return false + } + id := resource.Fields["id"] + if id == nil { + return false + } + return id.Reason == PositionalIDReason +} + // String returns a writable string leaf. encode writes a new string in the // syntax of the source. func String(value string, start, end int, encode func(string) ([]byte, error)) *Node { @@ -95,12 +127,38 @@ func Collections(collections map[string][]*Node) *Node { } // Resource returns one logical resource with its id field set, so that a -// keyed selector such as agents[id="support"] can find it. +// keyed selector such as agents[id="support"] can find it. A resource whose +// definition is not a literal object has only the id. func Resource(id string, fields map[string]*Node) *Node { + if fields == nil { + fields = make(map[string]*Node, 1) + } fields["id"] = &Node{Value: id} return &Node{Fields: fields} } +// Clone returns a deep copy of the node tree, so that a change to one use of +// a constant does not change another. The encoders are shared. +func (node *Node) Clone() *Node { + if node == nil { + return nil + } + copied := *node + if node.Fields != nil { + copied.Fields = make(map[string]*Node, len(node.Fields)) + for key, child := range node.Fields { + copied.Fields[key] = child.Clone() + } + } + if node.Items != nil { + copied.Items = make([]*Node, len(node.Items)) + for index, child := range node.Items { + copied.Items[index] = child.Clone() + } + } + return &copied +} + // IsCollection reports whether a path can index the node. func (node *Node) IsCollection() bool { if node == nil { @@ -182,10 +240,24 @@ func JSONEncoder(raw []byte) func(any) ([]byte, error) { return func(next any) ([]byte, error) { unit, prefix, multiline := jsonIndentation(raw) if !multiline { - return json.Marshal(next) + return EncodeJSON(next, "", "") } - return json.MarshalIndent(next, prefix, unit) + return EncodeJSON(next, prefix, unit) + } +} + +// EncodeJSON writes a value as JSON without the HTML escapes of the standard +// encoder, so that <, >, and & in prompt text stay as they are. A non-empty +// indent writes the value over several lines. +func EncodeJSON(value any, prefix, indent string) ([]byte, error) { + var buffer bytes.Buffer + encoder := json.NewEncoder(&buffer) + encoder.SetEscapeHTML(false) + encoder.SetIndent(prefix, indent) + if err := encoder.Encode(value); err != nil { + return nil, err } + return bytes.TrimRight(buffer.Bytes(), "\n"), nil } // jsonIndentation returns the indentation unit and the indentation of the diff --git a/internal/sync/adapter/reference.go b/internal/sync/adapter/reference.go index b20446631..4f94b86b2 100644 --- a/internal/sync/adapter/reference.go +++ b/internal/sync/adapter/reference.go @@ -3,6 +3,7 @@ package adapter import ( "fmt" "slices" + "strings" syncdomain "github.com/launchdarkly/ldcli/internal/sync" ) @@ -130,12 +131,22 @@ func fieldAllowed(kind syncdomain.Kind, field syncdomain.Field) bool { } } +// annotateProblem adds the resource and the file to the problem in an error. +// A wrapper such as `select "x": ...` names the part of the reference that +// failed, and the output prints the problem alone, so the problem takes that +// context into its message. func annotateProblem(err error, id syncdomain.ResourceID, file string) error { if err == nil { return nil } - if problem, ok := syncdomain.ProblemFrom(err); ok { - _ = problem.WithResource(id).WithFile(file, 0, 0) + problem, ok := syncdomain.ProblemFrom(err) + if !ok { + return err } - return err + if err != error(problem) { + if prefix, found := strings.CutSuffix(err.Error(), ": "+problem.Error()); found && prefix != "" { + problem.Message = prefix + ": " + problem.Message + } + } + return problem.WithResource(id).WithFile(file, 0, 0) } diff --git a/internal/sync/adapter/structured/json.go b/internal/sync/adapter/structured/json.go new file mode 100644 index 000000000..01c2adca1 --- /dev/null +++ b/internal/sync/adapter/structured/json.go @@ -0,0 +1,289 @@ +// Package structured inspects JSON, YAML, and TOML documents with exact byte +// spans for each scalar value. +package structured + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "unicode/utf8" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +type jsonAdapter struct{} + +func (jsonAdapter) Format() string { return "json" } +func (jsonAdapter) Behavior() string { return "json/1" } + +func (jsonAdapter) Inspect(bundle adapter.SourceBundle) (*adapter.Node, error) { + return parseJSONDocument(bundle.Primary) +} + +func (jsonAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "$", nil } + +func init() { adapter.MustRegister(jsonAdapter{}) } + +type jsonParser struct { + data []byte + offset int + path string +} + +func parseJSONDocument(document adapter.Document) (*adapter.Node, error) { + if !utf8.Valid(document.Data) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "JSON source is not valid UTF-8", + "Save the file as UTF-8.", + ).WithFile(document.Path, 0, 0) + } + parser := jsonParser{data: document.Data, path: document.Path} + if bytes.HasPrefix(parser.data, []byte("\ufeff")) { + parser.offset = len([]byte("\ufeff")) + } + parser.skipSpace() + root, err := parser.value() + if err != nil { + return nil, err + } + parser.skipSpace() + if parser.offset != len(parser.data) { + return nil, parser.problem("JSON source has trailing input") + } + return root, nil +} + +func (parser *jsonParser) value() (*adapter.Node, error) { + parser.skipSpace() + if parser.offset >= len(parser.data) { + return nil, parser.problem("JSON value is missing") + } + switch parser.data[parser.offset] { + case '{': + return parser.object() + case '[': + return parser.array() + case '"': + return parser.string() + case 't': + return parser.boolean("true") + case 'f': + return parser.boolean("false") + case 'n': + return parser.keyword("null", nil) + default: + return parser.number() + } +} + +func (parser *jsonParser) object() (*adapter.Node, error) { + start := parser.offset + parser.offset++ + parser.skipSpace() + fields := map[string]*adapter.Node{} + value := map[string]any{} + if parser.take('}') { + return parser.objectNode(value, fields, start, parser.offset), nil + } + for { + keyNode, err := parser.string() + if err != nil { + return nil, err + } + key := keyNode.Value.(string) + if _, duplicate := fields[key]; duplicate { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("JSON object contains duplicate key %q", key), + "Remove the duplicate key.", + ).WithFile(parser.path, 0, 0) + } + parser.skipSpace() + if !parser.take(':') { + return nil, parser.problem("JSON object key needs a colon") + } + item, err := parser.value() + if err != nil { + return nil, err + } + fields[key], value[key] = item, item.Value + parser.skipSpace() + if parser.take('}') { + return parser.objectNode(value, fields, start, parser.offset), nil + } + if !parser.take(',') { + return nil, parser.problem("JSON object needs a comma or closing brace") + } + parser.skipSpace() + } +} + +func (parser *jsonParser) array() (*adapter.Node, error) { + start := parser.offset + parser.offset++ + parser.skipSpace() + // An empty array is a value of its own, so it never reads as null. + items, values := []*adapter.Node{}, []any{} + if parser.take(']') { + return parser.arrayNode(values, items, start, parser.offset), nil + } + for { + item, err := parser.value() + if err != nil { + return nil, err + } + items, values = append(items, item), append(values, item.Value) + parser.skipSpace() + if parser.take(']') { + return parser.arrayNode(values, items, start, parser.offset), nil + } + if !parser.take(',') { + return nil, parser.problem("JSON array needs a comma or closing bracket") + } + parser.skipSpace() + } +} + +func (parser *jsonParser) string() (*adapter.Node, error) { + start := parser.offset + if !parser.take('"') { + return nil, parser.problem("JSON string must start with a quote") + } + escaped := false + for parser.offset < len(parser.data) { + next := parser.data[parser.offset] + parser.offset++ + switch { + case escaped: + escaped = false + case next == '\\': + escaped = true + case next == '"': + raw := parser.data[start:parser.offset] + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return nil, parser.problem("JSON string is invalid") + } + return adapter.String(value, start, parser.offset, func(text string) ([]byte, error) { + return adapter.EncodeJSON(text, "", "") + }), nil + case next < 0x20: + return nil, parser.problem("JSON string contains a control character") + } + } + return nil, parser.problem("JSON string is not closed") +} + +func (parser *jsonParser) number() (*adapter.Node, error) { + start := parser.offset + for parser.offset < len(parser.data) && bytes.ContainsRune([]byte("-+0123456789.eE"), rune(parser.data[parser.offset])) { + parser.offset++ + } + if start == parser.offset { + return nil, parser.problem("JSON value is invalid") + } + raw := parser.data[start:parser.offset] + var value float64 + if err := json.Unmarshal(raw, &value); err != nil { + return nil, parser.problem("JSON number is invalid") + } + // An integer beyond 53 bits has no exact float64, so it stays read-only + // text instead of a rounded number. + if !bytes.ContainsAny(raw, ".eE") { + if integer, err := strconv.ParseInt(string(raw), 10, 64); err != nil || !adapter.ExactInteger(integer) { + result := adapter.ReadOnly(start, parser.offset, "JSON integer is too large to represent exactly") + result.Value = string(raw) + return result, nil + } + } + return adapter.Number(value, start, parser.offset, func(number float64) ([]byte, error) { + return []byte(strconv.FormatFloat(number, 'g', -1, 64)), nil + }), nil +} + +// boolean reads true or false. The leaf accepts only a boolean on a write, +// as a string or a number leaf accepts only its own type. +func (parser *jsonParser) boolean(keyword string) (*adapter.Node, error) { + start := parser.offset + if !bytes.HasPrefix(parser.data[start:], []byte(keyword)) { + return nil, parser.problem("JSON value is invalid") + } + parser.offset += len(keyword) + return adapter.Bool(keyword == "true", start, parser.offset, nil), nil +} + +// keyword reads null. A null slot has no type of its own, so a write of any +// JSON value fills it. +func (parser *jsonParser) keyword(keyword string, value any) (*adapter.Node, error) { + start := parser.offset + if !bytes.HasPrefix(parser.data[start:], []byte(keyword)) { + return nil, parser.problem("JSON value is invalid") + } + parser.offset += len(keyword) + return &adapter.Node{ + Value: value, Start: start, End: parser.offset, Capability: adapter.CapabilityDirect, + Encode: func(next any) ([]byte, error) { return adapter.EncodeJSON(next, "", "") }, + }, nil +} + +func (parser *jsonParser) skipSpace() { + for parser.offset < len(parser.data) && bytes.ContainsRune([]byte(" \t\r\n"), rune(parser.data[parser.offset])) { + parser.offset++ + } +} + +func (parser *jsonParser) take(expected byte) bool { + if parser.offset >= len(parser.data) || parser.data[parser.offset] != expected { + return false + } + parser.offset++ + return true +} + +// problem returns an invalid-source problem that names the file, as the +// other adapters do, so a caller that does not annotate it still reports +// where the JSON failed. +func (parser *jsonParser) problem(message string) error { + return syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + fmt.Sprintf("%s at byte %d", message, parser.offset), + "Fix the JSON syntax.", + ).WithFile(parser.path, 0, 0) +} + +func (parser *jsonParser) objectNode(value map[string]any, fields map[string]*adapter.Node, start, end int) *adapter.Node { + result := &adapter.Node{ + Value: value, Fields: fields, Start: start, End: end, Capability: adapter.CapabilityDirect, + Encode: adapter.JSONEncoder(parser.data[start:end]), + } + for _, field := range fields { + jsonContainerHolds(result, field) + } + return result +} + +func (parser *jsonParser) arrayNode(value []any, items []*adapter.Node, start, end int) *adapter.Node { + result := &adapter.Node{ + Value: value, Items: items, Collection: true, Start: start, End: end, Capability: adapter.CapabilityDirect, + Encode: adapter.JSONEncoder(parser.data[start:end]), + } + for _, item := range items { + jsonContainerHolds(result, item) + } + return result +} + +// jsonContainerHolds makes a container read-only when a value in it is, such +// as an integer that a float64 cannot hold exactly. A write of the whole +// container would replace that value with a rounded one. +func jsonContainerHolds(container, child *adapter.Node) { + if child == nil || child.Capability == adapter.CapabilityDirect || container.Capability != adapter.CapabilityDirect { + return + } + container.Capability = adapter.CapabilityReadOnly + container.Reason = "JSON container holds a read-only value: " + child.Reason + container.Encode = nil +} diff --git a/internal/sync/adapter/structured/problem.go b/internal/sync/adapter/structured/problem.go new file mode 100644 index 000000000..17bc0f52b --- /dev/null +++ b/internal/sync/adapter/structured/problem.go @@ -0,0 +1,12 @@ +package structured + +import syncdomain "github.com/launchdarkly/ldcli/internal/sync" + +// nameDocument puts the path of the document on a problem that has none, so +// that a caller that does not annotate the problem still reports the file. +func nameDocument(err error, path string) error { + if problem, ok := syncdomain.ProblemFrom(err); ok && problem.File == "" { + _ = problem.WithFile(path, 0, 0) + } + return err +} diff --git a/internal/sync/adapter/structured/structured_test.go b/internal/sync/adapter/structured/structured_test.go new file mode 100644 index 000000000..7d7e87a4b --- /dev/null +++ b/internal/sync/adapter/structured/structured_test.go @@ -0,0 +1,304 @@ +package structured + +import ( + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" + "github.com/launchdarkly/ldcli/internal/sync/adapter/yamltree" +) + +func TestJSONAdapterRejectsMalformedSources(t *testing.T) { + inputs := []string{ + "", + `{"key" "value"}`, + `{"key": true "next": false}`, + `{"key": 1, "key": 2}`, + `[1 2]`, + `{"key":}`, + `{"key":"unterminated}`, + "{\"key\":\"control\x01\"}", + `truth`, + `+`, + `1e`, + `true trailing`, + } + for _, input := range inputs { + t.Run(input, func(t *testing.T) { + _, err := parseJSONDocument(adapter.Document{Path: "value.json", Data: []byte(input)}) + require.Error(t, err) + }) + } + _, err := parseJSONDocument(adapter.Document{Path: "value.json", Data: []byte{0xff}}) + require.Error(t, err) + + root, err := parseJSONDocument(adapter.Document{Path: "value.json", Data: []byte("\ufeff {\"empty\":{},\"array\":[],\"false\":false,\"null\":null}")}) + require.NoError(t, err) + assert.Equal(t, false, root.Fields["false"].Value) +} + +func TestYAMLAdapterHandlesSafetyForms(t *testing.T) { + for name, input := range map[string][]byte{ + "invalid UTF-8": {0xff}, + "invalid syntax": []byte("value: ["), + "empty": nil, + "non-string key": []byte("1: value"), + "duplicate": []byte("key: one\nkey: two\n"), + } { + t.Run(name, func(t *testing.T) { + _, err := parseYAMLDocument(adapter.Document{Path: "value.yaml", Data: input}) + require.Error(t, err) + }) + } + + root, err := parseYAMLDocument(adapter.Document{Path: "value.yaml", Data: []byte(` +defaults: &defaults + enabled: true + count: 1_000 +copy: + <<: *defaults +tagged: !custom value +block: | + content +sequence: &items [one, two] +null_value: null +--- +second: document +`)}) + require.NoError(t, err) + assert.Equal(t, adapter.CapabilityReadOnly, root.Capability) + assert.Equal(t, adapter.CapabilityReadOnly, root.Fields["copy"].Capability) + assert.Equal(t, adapter.CapabilityReadOnly, root.Fields["tagged"].Capability) +} + +func TestTOMLAdapterHandlesSafetyForms(t *testing.T) { + for name, input := range map[string][]byte{ + "invalid UTF-8": {0xff}, + "invalid syntax": []byte("value = ["), + "duplicate": []byte("value = 1\nvalue = 2\n"), + "value as table": []byte("value = 1\n[value]\nchild = 2\n"), + } { + t.Run(name, func(t *testing.T) { + _, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: input}) + require.Error(t, err) + }) + } + + root, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte("\ufeff" + ` +basic = "text" +literal = 'text' +multiline_basic = """text""" +multiline_literal = '''text''' +enabled = true +integer = 10 +float = 1.5 +date = 2026-10-30 +array = [1, # comment + 2] +inline = { name = "value" } +[[tools]] +name = "one" +[[tools]] +name = "two" +`)}) + require.NoError(t, err) + assert.Equal(t, "2026-10-30", root.Fields["date"].Value) + assert.Len(t, root.Fields["tools"].Items, 2) + + assert.Equal(t, `"""next\"\"\"value"""`, mustTOMLString(t, `next"""value`, `"""old"""`)) + assert.Equal(t, "'''next'''", mustTOMLString(t, "next", "'''old'''")) + _, err = encodeTOMLString("bad'''value", "'''old'''") + require.Error(t, err) + _, err = encodeTOMLString("bad\nvalue", "'old'") + require.Error(t, err) + assert.Equal(t, `"\b\t\n\f\r\u0001\u0080\"\\"`, mustTOMLString(t, "\b\t\n\f\r\x01\u0080\"\\", `"old"`)) +} + +func TestYAMLStringCodecRejectsUnsupportedCodePoint(t *testing.T) { + _, err := yamltree.EncodeString("\uffff", "plain") + require.ErrorContains(t, err, "unsupported character U+FFFF") +} + +func mustTOMLString(t *testing.T, value, previous string) string { + t.Helper() + encoded, err := encodeTOMLString(value, previous) + require.NoError(t, err) + return string(encoded) +} + +func FuzzTOMLStringCodec(f *testing.F) { + for _, value := range []string{"plain", "quotes \" and slash \\", "line one\nline two", "Unicode 🌍"} { + f.Add(value) + } + f.Fuzz(func(t *testing.T, value string) { + if !utf8.ValidString(value) { + t.Skip() + } + encoded, err := encodeTOMLString(value, `"plain"`) + require.NoError(t, err) + source := append([]byte("value = "), encoded...) + source = append(source, '\n') + root, err := adapter.ParseDocument(adapter.Document{Path: "value.toml", Data: source}, "toml") + require.NoError(t, err) + node, err := root.Resolve("value") + require.NoError(t, err) + require.Equal(t, value, node.Value) + }) +} + +func TestYAMLTrailingDocumentMarkerKeepsTheDocumentWritable(t *testing.T) { + root, err := parseYAMLDocument(adapter.Document{Path: "value.yaml", Data: []byte("name: one\n---\n")}) + require.NoError(t, err) + assert.Equal(t, adapter.CapabilityDirect, root.Fields["name"].Capability) + + root, err = parseYAMLDocument(adapter.Document{Path: "value.yaml", Data: []byte("name: one\n---\nname: two\n")}) + require.NoError(t, err) + assert.Equal(t, adapter.CapabilityReadOnly, root.Fields["name"].Capability, "a second document with content is read-only") +} + +func TestYAMLLaterDocumentsAfterAnEmptyOneAreMultiDocument(t *testing.T) { + root, err := parseYAMLDocument(adapter.Document{Path: "value.yaml", Data: []byte("name: one\n---\n---\nname: two\n")}) + require.NoError(t, err) + assert.Equal(t, adapter.CapabilityReadOnly, root.Fields["name"].Capability) +} + +func TestJSONWritesKeepHTMLCharacters(t *testing.T) { + root, err := parseJSONDocument(adapter.Document{Path: "value.json", Data: []byte(`{"prompt": "a", "schema": {"type": "object"}}`)}) + require.NoError(t, err) + encoded, err := root.Fields["prompt"].Encode("Use bold & more") + require.NoError(t, err) + assert.Equal(t, `"Use bold & more"`, string(encoded)) + encoded, err = root.Fields["schema"].Encode(map[string]any{"description": "a < b"}) + require.NoError(t, err) + assert.Equal(t, `{"description":"a < b"}`, string(encoded)) +} + +func TestEmptyArraysAreNotNull(t *testing.T) { + jsonRoot, err := parseJSONDocument(adapter.Document{Path: "value.json", Data: []byte(`{"items": [], "none": null}`)}) + require.NoError(t, err) + assert.Equal(t, []any{}, jsonRoot.Fields["items"].Value) + assert.NotNil(t, jsonRoot.Fields["items"].Value, "an empty JSON array is a value, not null") + assert.Nil(t, jsonRoot.Fields["none"].Value) + assert.Equal(t, map[string]any{"items": []any{}, "none": nil}, jsonRoot.Value) + + tomlRoot, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte("items = []\n")}) + require.NoError(t, err) + assert.Equal(t, []any{}, tomlRoot.Fields["items"].Value) +} + +func TestJSONBooleanWritesCheckTheType(t *testing.T) { + root, err := parseJSONDocument(adapter.Document{Path: "value.json", Data: []byte(`{"flag": true, "slot": null}`)}) + require.NoError(t, err) + flag := root.Fields["flag"] + assert.Equal(t, true, flag.Value) + _, err = flag.Encode("yes") + require.Error(t, err, "a boolean leaf rejects a string") + encoded, err := flag.Encode(false) + require.NoError(t, err) + assert.Equal(t, "false", string(encoded)) + + encoded, err = root.Fields["slot"].Encode("text") + require.NoError(t, err, "a null slot takes any JSON value") + assert.Equal(t, `"text"`, string(encoded)) +} + +func TestJSONIntegersBeyondExactFloatsAreReadOnly(t *testing.T) { + root, err := parseJSONDocument(adapter.Document{Path: "value.json", Data: []byte( + `{"exact": 9007199254740992, "big": 9007199254740993, "huge": 99999999999999999999, "negative": -9007199254740993, "ratio": 1.5, "exponent": 1e20}`, + )}) + require.NoError(t, err) + assert.Equal(t, float64(1<<53), root.Fields["exact"].Value, "the largest exact integer is a number") + for _, field := range []string{"big", "huge", "negative"} { + value := root.Fields[field] + assert.Equal(t, adapter.CapabilityReadOnly, value.Capability, "%s has no exact float64", field) + assert.Equal(t, "JSON integer is too large to represent exactly", value.Reason, field) + } + assert.Equal(t, "9007199254740993", root.Fields["big"].Value, "the text of the integer is kept") + assert.Equal(t, 1.5, root.Fields["ratio"].Value) + assert.Equal(t, 1e20, root.Fields["exponent"].Value, "a float literal keeps its float value") + assert.Equal(t, adapter.CapabilityReadOnly, root.Capability, "the object that holds the integer is read-only as a whole") + assert.Nil(t, root.Encode) + + nested, err := parseJSONDocument(adapter.Document{Path: "value.json", Data: []byte(`{"schema": {"limits": [9007199254740993]}, "name": "ok"}`)}) + require.NoError(t, err) + assert.Equal(t, adapter.CapabilityReadOnly, nested.Fields["schema"].Capability, "a nested object that holds the integer is read-only") + assert.Equal(t, adapter.CapabilityReadOnly, nested.Fields["schema"].Fields["limits"].Capability, "the array that holds the integer is read-only") + assert.Equal(t, adapter.CapabilityDirect, nested.Fields["name"].Capability, "a sibling leaf stays writable") +} + +func TestYAMLIntegersBeyondExactFloatsAreReadOnly(t *testing.T) { + root, err := parseYAMLDocument(adapter.Document{Path: "value.yaml", Data: []byte( + "exact: 9007199254740992\nbig: 9007199254740993\nhuge: 99999999999999999999\nnegative: -9007199254740993\nratio: 1.5\n", + )}) + require.NoError(t, err) + assert.Equal(t, float64(1<<53), root.Fields["exact"].Value, "the largest exact integer is a number") + for _, field := range []string{"big", "huge", "negative"} { + value := root.Fields[field] + assert.Equal(t, adapter.CapabilityReadOnly, value.Capability, "%s has no exact float64", field) + assert.Equal(t, "YAML integer is too large to represent exactly", value.Reason, field) + } + assert.Equal(t, "9007199254740993", root.Fields["big"].Value, "the text of the integer is kept") + assert.Equal(t, 1.5, root.Fields["ratio"].Value) + + encoded, err := root.Fields["exact"].Encode(float64(1000000)) + require.NoError(t, err) + assert.Equal(t, "1000000", string(encoded), "an integer slot writes an integer, not scientific notation") + _, err = root.Fields["exact"].Encode(1.5) + require.Error(t, err, "an integer slot rejects a fraction") + encoded, err = root.Fields["ratio"].Encode(float64(1000000)) + require.NoError(t, err) + assert.Equal(t, "1e+06", string(encoded), "a float slot keeps the float spelling") + encoded, err = root.Fields["ratio"].Encode(float64(2)) + require.NoError(t, err) + assert.Equal(t, "2.0", string(encoded), "a whole number in a float slot keeps a fraction") +} + +func TestYAMLWritesKeepHTMLCharacters(t *testing.T) { + root, err := parseYAMLDocument(adapter.Document{Path: "value.yaml", Data: []byte("prompt: \"a < b\"\nplain: text\n")}) + require.NoError(t, err) + encoded, err := root.Fields["prompt"].Encode("x > y & z") + require.NoError(t, err) + assert.Equal(t, `"x > y & z"`, string(encoded), "a quoted YAML string keeps <, >, and &") + encoded, err = root.Fields["plain"].Encode("a: b & c") + require.NoError(t, err) + assert.Equal(t, `"a: b & c"`, string(encoded)) +} + +func TestJSONProblemsNameTheFile(t *testing.T) { + for name, data := range map[string]string{ + "syntax": `{"a": 1,`, + "duplicate": `{"a": 1, "a": 2}`, + "trailing": `{} x`, + } { + _, err := parseJSONDocument(adapter.Document{Path: "config.json", Data: []byte(data)}) + require.Error(t, err, name) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok, name) + assert.Equal(t, "config.json", problem.File, "the %s problem names the file", name) + } +} + +func TestYAMLAndTOMLProblemsNameTheFile(t *testing.T) { + cases := map[string]func() (*adapter.Node, error){ + "yaml empty": func() (*adapter.Node, error) { + return parseYAMLDocument(adapter.Document{Path: "config.yaml", Data: []byte("")}) + }, + "yaml duplicate": func() (*adapter.Node, error) { + return parseYAMLDocument(adapter.Document{Path: "config.yaml", Data: []byte("a: 1\na: 2\n")}) + }, + "toml duplicate": func() (*adapter.Node, error) { + return parseTOMLDocument(adapter.Document{Path: "config.yaml", Data: []byte("a = 1\na = 2\n")}) + }, + } + for name, parse := range cases { + _, err := parse() + require.Error(t, err, name) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok, name) + assert.Equal(t, "config.yaml", problem.File, "the %s problem names the file", name) + } +} diff --git a/internal/sync/adapter/structured/testdata/agents.yaml b/internal/sync/adapter/structured/testdata/agents.yaml new file mode 100644 index 000000000..2bb224b47 --- /dev/null +++ b/internal/sync/adapter/structured/testdata/agents.yaml @@ -0,0 +1,16 @@ +service: support +agents: + - name: support + prompt: "Ask for an order ID." + model: + name: gpt-5 + temperature: 0.2 + enabled: true + - name: refunds + prompt: 'Explain the refund policy.' + model: + name: gpt-5 + temperature: 0.1 + enabled: false +metadata: + "display.name": Support agents diff --git a/internal/sync/adapter/structured/testdata/mcp.json b/internal/sync/adapter/structured/testdata/mcp.json new file mode 100644 index 000000000..341469504 --- /dev/null +++ b/internal/sync/adapter/structured/testdata/mcp.json @@ -0,0 +1,32 @@ +{ + "server": "orders", + "tools": [ + { + "name": "lookup_order", + "description": "Look up one order.", + "inputSchema": { + "type": "object", + "properties": { + "orderId": { "type": "string" } + }, + "required": ["orderId"] + } + }, + { + "name": "refund_order", + "description": "Refund one order.", + "inputSchema": { + "type": "object", + "properties": { + "orderId": { "type": "string" }, + "reason": { "type": "string" } + }, + "required": ["orderId", "reason"] + } + } + ], + "metadata": { + "display.name": "Order tools", + "enabled": true + } +} diff --git a/internal/sync/adapter/structured/testdata/pyproject.toml b/internal/sync/adapter/structured/testdata/pyproject.toml new file mode 100644 index 000000000..689ed0c28 --- /dev/null +++ b/internal/sync/adapter/structured/testdata/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name = "support-agents" +version = "1.0.0" + +[tool.agent.support] +prompt = """Ask for an order ID. +Keep the answer short.""" +model = "gpt-5" +temperature = 0.2 +enabled = true + +[tool.agent.refunds] +prompt = 'Explain the refund policy.' +model = 'gpt-5' +temperature = 0.1 +enabled = false + +[[tool.agent.plugins]] +name = "audit" +enabled = true diff --git a/internal/sync/adapter/structured/toml.go b/internal/sync/adapter/structured/toml.go new file mode 100644 index 000000000..a0f8edd91 --- /dev/null +++ b/internal/sync/adapter/structured/toml.go @@ -0,0 +1,490 @@ +package structured + +import ( + "bytes" + "fmt" + "math" + "strconv" + "strings" + "unicode/utf8" + + toml "github.com/pelletier/go-toml/v2" + "github.com/pelletier/go-toml/v2/unstable" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +type tomlAdapter struct{} + +func (tomlAdapter) Format() string { return "toml" } +func (tomlAdapter) Behavior() string { return "toml/1" } + +func (tomlAdapter) Inspect(bundle adapter.SourceBundle) (*adapter.Node, error) { + return parseTOMLDocument(bundle.Primary) +} + +func (tomlAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "$", nil } + +func init() { adapter.MustRegister(tomlAdapter{}) } + +func parseTOMLDocument(document adapter.Document) (*adapter.Node, error) { + if !utf8.Valid(document.Data) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "TOML source is not valid UTF-8", + "Save the file as UTF-8.", + ).WithFile(document.Path, 0, 0) + } + data := document.Data + offset := 0 + if bytes.HasPrefix(data, []byte("\ufeff")) { + offset = len([]byte("\ufeff")) + data = data[offset:] + } + var parser unstable.Parser + parser.Reset(data) + root := newTOMLTable(nil) + table := root + tree := newTOMLTree() + for parser.NextExpression() { + expression := parser.Expression() + switch expression.Kind { + case unstable.KeyValue: + keys := tomlKeys(expression) + if err := addTOMLValue(data, offset, table, keys, expression.Value(), tree); err != nil { + return nil, nameDocument(err, document.Path) + } + case unstable.Table: + var err error + table, err = tree.table(root, tomlKeys(expression)) + if err != nil { + return nil, nameDocument(err, document.Path) + } + case unstable.ArrayTable: + var err error + table, err = tree.arrayTable(root, tomlKeys(expression)) + if err != nil { + return nil, nameDocument(err, document.Path) + } + } + } + if err := parser.Error(); err != nil { + return nil, syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "TOML source is invalid", "Fix the TOML syntax.", + ).WithFile(document.Path, 0, 0), err) + } + return root, nil +} + +func tomlKeys(node *unstable.Node) []string { + var keys []string + iterator := node.Key() + for iterator.Next() { + keys = append(keys, string(iterator.Node().Data)) + } + return keys +} + +// tomlTree records how each table of a document was defined, because TOML +// limits what can extend it. A [header] defines a table once, and a later +// header can add a sub-table to it. A dotted key cannot add to a table that a +// header defined. Nothing can add to an inline table. +type tomlTree struct { + // arrayTables are the arrays that [[header]] lines build. + arrayTables map[*adapter.Node]bool + // headers are the tables that a [header] or a [[header]] item defined. + headers map[*adapter.Node]bool + // implicit are the tables that a header path created on its way, as + // [a.b.c] creates a and a.b. A later header can define one of them once, + // and a dotted key cannot extend it. + implicit map[*adapter.Node]bool + // dotted are the tables that a dotted key created. + dotted map[*adapter.Node]bool + // inline are the inline tables. + inline map[*adapter.Node]bool +} + +func newTOMLTree() *tomlTree { + return &tomlTree{ + arrayTables: map[*adapter.Node]bool{}, headers: map[*adapter.Node]bool{}, + dotted: map[*adapter.Node]bool{}, inline: map[*adapter.Node]bool{}, implicit: map[*adapter.Node]bool{}, + } +} + +// addTOMLValue adds one key and value to a table. An inline table has a tree +// of its own, because no header can open it again and its own keys follow +// the same extension rules. +func addTOMLValue( + data []byte, + offset int, + table *adapter.Node, + keys []string, + valueNode *unstable.Node, + tree *tomlTree, +) error { + if len(keys) == 0 { + return syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "TOML key is missing", "Add a key to the value.", + ) + } + parent, err := tree.objectPath(table, keys[:len(keys)-1], true) + if err != nil { + return err + } + key := keys[len(keys)-1] + if _, duplicate := parent.Fields[key]; duplicate { + return syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("TOML table contains duplicate key %q", strings.Join(keys, ".")), + "Remove the duplicate key.", + ) + } + value, err := tomlValueNode(data, offset, valueNode) + if err != nil { + return err + } + if tomlFrozen(table) { + value.MarkReadOnly(table.Reason) + } + if tree != nil && valueNode.Kind == unstable.InlineTable { + tree.inline[value] = true + } + parent.Fields[key] = value + parent.Value.(map[string]any)[key] = value.Value + return nil +} + +const ( + tomlArrayTableReason = "TOML arrays of tables are read-only" + tomlStructureReason = "TOML structural changes are read-only" +) + +// tomlFrozen reports whether a table and every value in it are read-only, +// as an item of an array of tables is. A table that is only structurally +// read-only keeps its scalar values writable. +func tomlFrozen(table *adapter.Node) bool { + return table.Capability == adapter.CapabilityReadOnly && table.Reason == tomlArrayTableReason +} + +// table returns the table that a [table] header opens. A header can define a +// table once, and cannot open an array of tables, an inline table, or a +// table that a dotted key defined. +func (tree *tomlTree) table(root *adapter.Node, keys []string) (*adapter.Node, error) { + if len(keys) == 0 { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "TOML table key is missing", "Add a table key.", + ) + } + parent, err := tree.objectPath(root, keys[:len(keys)-1], false) + if err != nil { + return nil, err + } + key := keys[len(keys)-1] + table := parent.Fields[key] + switch { + case table == nil: + table = newTOMLTable(parent) + parent.Fields[key] = table + parent.Value.(map[string]any)[key] = table.Value + case table.Fields == nil || tree.arrayTables[table] || tree.headers[table] || tree.dotted[table] || tree.inline[table]: + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("TOML table %q is defined more than once", strings.Join(keys, ".")), + "Keep one table definition.", + ) + } + tree.headers[table] = true + delete(tree.implicit, table) + return table, nil +} + +// arrayTable returns the new item that a [[table]] header appends to its +// array of tables. A key that already holds a value, a table, or a plain +// array cannot become an array of tables. +func (tree *tomlTree) arrayTable(root *adapter.Node, keys []string) (*adapter.Node, error) { + if len(keys) == 0 { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "TOML array table key is missing", "Add a table key.", + ) + } + parent, err := tree.objectPath(root, keys[:len(keys)-1], false) + if err != nil { + return nil, err + } + key := keys[len(keys)-1] + collection := parent.Fields[key] + if collection == nil { + collection = &adapter.Node{ + Value: []any{}, Collection: true, Capability: adapter.CapabilityReadOnly, + Reason: tomlArrayTableReason, + } + tree.arrayTables[collection] = true + parent.Fields[key] = collection + parent.Value.(map[string]any)[key] = collection.Value + } + if !tree.arrayTables[collection] { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("TOML key %q has incompatible definitions", strings.Join(keys, ".")), + "Keep one table definition.", + ) + } + item := &adapter.Node{ + Value: map[string]any{}, Fields: map[string]*adapter.Node{}, + Capability: adapter.CapabilityReadOnly, Reason: tomlArrayTableReason, + } + tree.headers[item] = true + collection.Items = append(collection.Items, item) + collection.Value = append(collection.Value.([]any), item.Value) + parent.Value.(map[string]any)[key] = collection.Value + return item, nil +} + +// newTOMLTable returns an empty table under parent, or the root table when +// parent is nil. Sync edits the values of a table and never rewrites the +// table as a whole, so the table is structurally read-only. A table inside +// an item of an array of tables is read-only with its values, as the item is. +func newTOMLTable(parent *adapter.Node) *adapter.Node { + table := &adapter.Node{ + Value: map[string]any{}, Fields: map[string]*adapter.Node{}, + Capability: adapter.CapabilityReadOnly, Reason: tomlStructureReason, + } + if parent != nil && tomlFrozen(parent) { + table.Reason = parent.Reason + } + return table +} + +// objectPath returns the table at a dotted key path and creates the tables +// that do not exist yet. A header path through an array of tables continues +// in its last item, as [servers.limits] after [[servers]] does. A dotted key +// cannot extend a table that a header defined or an array of tables, and no +// path can extend an inline table. +func (tree *tomlTree) objectPath(root *adapter.Node, keys []string, dotted bool) (*adapter.Node, error) { + current := root + for index, key := range keys { + child := current.Fields[key] + if child == nil { + child = newTOMLTable(current) + current.Fields[key] = child + current.Value.(map[string]any)[key] = child.Value + if tree != nil { + if dotted { + tree.dotted[child] = true + } else { + tree.implicit[child] = true + } + } + } + if tree != nil && (tree.inline[child] || + dotted && (tree.headers[child] || tree.arrayTables[child] || tree.implicit[child])) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("TOML table %q cannot be extended here", strings.Join(keys[:index+1], ".")), + "Define the keys where the table is defined.", + ) + } + if tree != nil && tree.arrayTables[child] && len(child.Items) != 0 { + child = child.Items[len(child.Items)-1] + } + if child.Fields == nil { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("TOML key %q is already a value", key), + "Keep one compatible key definition.", + ) + } + current = child + } + return current, nil +} + +func tomlValueNode(data []byte, offset int, node *unstable.Node) (*adapter.Node, error) { + start := offset + int(node.Raw.Offset) + end := start + int(node.Raw.Length) + raw := data[node.Raw.Offset : node.Raw.Offset+node.Raw.Length] + switch node.Kind { + case unstable.String: + return adapter.String(string(node.Data), start, end, func(value string) ([]byte, error) { + return encodeTOMLString(value, string(raw)) + }), nil + case unstable.Bool: + return adapter.Bool(string(node.Data) == "true", start, end, nil), nil + case unstable.Integer, unstable.Float: + // An integer beyond int64 fails to decode, so it stays read-only + // text, as an integer beyond 53 bits does, and the rest of the + // document stays usable. + if node.Kind == unstable.Integer { + if _, err := strconv.ParseInt(strings.ReplaceAll(string(raw), "_", ""), 0, 64); err != nil { + result := adapter.ReadOnly(start, end, "TOML integer is too large to represent exactly") + result.Value = string(raw) + return result, nil + } + } + var decoded map[string]any + if err := toml.Unmarshal(append([]byte("value = "), raw...), &decoded); err != nil { + return nil, err + } + // An integer beyond 53 bits has no exact float64, so it stays + // read-only text instead of a rounded number. + if integer, isInteger := decoded["value"].(int64); isInteger && !adapter.ExactInteger(integer) { + result := adapter.ReadOnly(start, end, "TOML integer is too large to represent exactly") + result.Value = string(raw) + return result, nil + } + number, ok := adapter.AsNumber(decoded["value"]) + if !ok { + return adapter.ReadOnly(start, end, "TOML number is not representable"), nil + } + // Infinity and NaN have no JSON form, so they stay read-only text. + if math.IsInf(number, 0) || math.IsNaN(number) { + result := adapter.ReadOnly(start, end, "TOML infinity and NaN values are read-only") + result.Value = string(raw) + return result, nil + } + return adapter.Number(number, start, end, tomlNumberEncoder(node.Kind == unstable.Integer)), nil + case unstable.Array: + return tomlArrayNode(data, offset, node) + case unstable.InlineTable: + return tomlInlineTableNode(data, offset, node) + default: + result := adapter.ReadOnly(start, end, "TOML date and time values are read-only") + result.Value = string(node.Data) + return result, nil + } +} + +func tomlArrayNode(data []byte, offset int, node *unstable.Node) (*adapter.Node, error) { + items, values := []*adapter.Node{}, []any{} + iterator := node.Children() + for iterator.Next() { + child := iterator.Node() + if child.Kind == unstable.Comment { + continue + } + item, err := tomlValueNode(data, offset, child) + if err != nil { + return nil, err + } + items, values = append(items, item), append(values, item.Value) + } + return &adapter.Node{ + Value: values, Items: items, Collection: true, Capability: adapter.CapabilityReadOnly, + Reason: tomlStructureReason, + }, nil +} + +func tomlInlineTableNode(data []byte, offset int, node *unstable.Node) (*adapter.Node, error) { + result := newTOMLTable(nil) + tree := newTOMLTree() + iterator := node.Children() + for iterator.Next() { + child := iterator.Node() + if child.Kind != unstable.KeyValue { + continue + } + if err := addTOMLValue(data, offset, result, tomlKeys(child), child.Value(), tree); err != nil { + return nil, err + } + } + return result, nil +} + +// tomlNumberEncoder writes a number in the type of the value that it +// replaces. TOML tells an integer from a float, so an integer slot takes a +// whole number only and a float slot keeps a fraction or a trailing .0. +func tomlNumberEncoder(integer bool) func(float64) ([]byte, error) { + return func(next float64) ([]byte, error) { + if integer { + if next != math.Trunc(next) || next > float64(adapter.MaxExactInteger) || next < -float64(adapter.MaxExactInteger) { + return nil, fmt.Errorf("TOML integer cannot hold %v", next) + } + return []byte(strconv.FormatInt(int64(next), 10)), nil + } + text := strconv.FormatFloat(next, 'g', -1, 64) + if !strings.ContainsAny(text, ".eE") { + text += ".0" + } + return []byte(text), nil + } +} + +// encodeTOMLString writes a value in the string form of the text that it +// replaces. A literal string has no escapes, so it cannot hold every value. +func encodeTOMLString(value, previous string) ([]byte, error) { + switch { + case strings.HasPrefix(previous, `"""`): + return []byte(tomlMultilineOpening(`"""`, previous, value) + encodeTOMLBasicBody(value, true) + `"""`), nil + case strings.HasPrefix(previous, `'''`): + if strings.Contains(value, `'''`) || !tomlLiteralCanHold(value, true) { + return nil, fmt.Errorf("multiline literal string cannot represent the new value") + } + return []byte(tomlMultilineOpening(`'''`, previous, value) + value + `'''`), nil + case strings.HasPrefix(previous, `'`): + if strings.Contains(value, `'`) || !tomlLiteralCanHold(value, false) { + return nil, fmt.Errorf("literal string cannot represent the new value") + } + return []byte("'" + value + "'"), nil + default: + return []byte(`"` + encodeTOMLBasicBody(value, false) + `"`), nil + } +} + +// tomlMultilineOpening returns the opening delimiter of a multiline string. +// TOML trims a newline right after the delimiter, so a value that starts with +// a newline needs one more. A source that had one keeps its layout. +func tomlMultilineOpening(delimiter, previous, value string) string { + rest := strings.TrimPrefix(previous, delimiter) + if strings.HasPrefix(value, "\n") || strings.HasPrefix(value, "\r\n") || + strings.HasPrefix(rest, "\n") || strings.HasPrefix(rest, "\r\n") { + return delimiter + "\n" + } + return delimiter +} + +// tomlLiteralCanHold reports whether a literal string can hold a value. A +// literal string allows a tab but no other control character. A multiline +// literal string also allows a newline. +func tomlLiteralCanHold(value string, multiline bool) bool { + for index := 0; index < len(value); index++ { + switch character := value[index]; { + case character == '\t' || character >= 0x20 && character != 0x7f: + case multiline && character == '\n': + case multiline && character == '\r' && index+1 < len(value) && value[index+1] == '\n': + default: + return false + } + } + return true +} + +// encodeTOMLBasicBody escapes a value for a basic string. A multiline basic +// string keeps a newline and a tab as they are. +func encodeTOMLBasicBody(value string, multiline bool) string { + var encoded strings.Builder + for _, character := range value { + switch { + case character == '"' || character == '\\': + encoded.WriteByte('\\') + encoded.WriteRune(character) + case multiline && (character == '\n' || character == '\t'): + encoded.WriteRune(character) + case character == '\b': + encoded.WriteString(`\b`) + case character == '\t': + encoded.WriteString(`\t`) + case character == '\n': + encoded.WriteString(`\n`) + case character == '\f': + encoded.WriteString(`\f`) + case character == '\r': + encoded.WriteString(`\r`) + case character < 0x20 || character >= 0x7f && character <= 0x9f: + fmt.Fprintf(&encoded, `\u%04X`, character) + default: + encoded.WriteRune(character) + } + } + return encoded.String() +} diff --git a/internal/sync/adapter/structured/toml_test.go b/internal/sync/adapter/structured/toml_test.go new file mode 100644 index 000000000..3b9ee1954 --- /dev/null +++ b/internal/sync/adapter/structured/toml_test.go @@ -0,0 +1,218 @@ +package structured + +import ( + "testing" + + toml "github.com/pelletier/go-toml/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +func TestTOMLArrayTablesRejectAnIncompatibleKey(t *testing.T) { + for name, input := range map[string]string{ + "value": "tools = 1\n[[tools]]\nname = \"one\"\n", + "table": "[tools]\nname = \"one\"\n[[tools]]\nname = \"two\"\n", + "inline table": "tools = { name = \"one\" }\n[[tools]]\nname = \"two\"\n", + "plain array": "tools = []\n[[tools]]\nname = \"one\"\n", + } { + t.Run(name, func(t *testing.T) { + _, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte(input)}) + require.ErrorContains(t, err, `TOML key "tools" has incompatible definitions`) + }) + } +} + +func TestTOMLSubtableOfAnArrayTableJoinsTheLastItem(t *testing.T) { + root, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte(` +[[servers]] +name = "one" +[servers.limits] +max = 1 +[[servers]] +name = "two" +`)}) + require.NoError(t, err) + + servers := root.Fields["servers"] + require.Len(t, servers.Items, 2) + assert.True(t, servers.Collection) + assert.Equal(t, 1.0, servers.Items[0].Fields["limits"].Fields["max"].Value) + assert.Nil(t, servers.Items[1].Fields["limits"]) + assert.Equal(t, []any{ + map[string]any{"name": "one", "limits": map[string]any{"max": 1.0}}, + map[string]any{"name": "two"}, + }, root.Value.(map[string]any)["servers"]) +} + +func TestTOMLStringEncodersRoundTrip(t *testing.T) { + values := map[string]string{ + "escape": `path C:\new\table`, + "leading newline": "\nfirst\nsecond", + "trailing backslash": `ends with \`, + "quotes": `say """hi""" now"`, + "control": "tab\tbell\a", + "crlf": "one\r\ntwo", + "unicode": "caf\u00e9 \u2014 ok", + } + previous := []string{`"old"`, `"""old"""`, "\"\"\"\nold\"\"\"", `'old'`, `'''old'''`, "'''\nold'''"} + for name, value := range values { + for _, old := range previous { + t.Run(name+" after "+old, func(t *testing.T) { + encoded, err := encodeTOMLString(value, old) + if err != nil { + // A literal string has no escapes, so it cannot hold a + // control character, and a single-line one no newline. + require.True(t, old[0] == '\'', "basic strings hold every value") + return + } + var decoded struct { + Value string `toml:"value"` + } + require.NoError(t, toml.Unmarshal([]byte("value = "+string(encoded)+"\n"), &decoded), string(encoded)) + assert.Equal(t, value, decoded.Value) + }) + } + } +} + +func TestTOMLHeadersDefineATableOnce(t *testing.T) { + for name, input := range map[string]string{ + "table twice": "[a]\nx = 1\n[a]\ny = 2\n", + "table after array": "[[a]]\nx = 1\n[a]\ny = 2\n", + "table after inline": "a = { x = 1 }\n[a]\ny = 2\n", + "table after dotted key": "a.x = 1\n[a]\ny = 2\n", + "dotted key after table": "[a]\nx = 1\n[b]\na.y = 2\n[a]\nz = 3\n", + } { + t.Run(name, func(t *testing.T) { + _, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte(input)}) + require.ErrorContains(t, err, `is defined more than once`) + }) + } + + root, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte(` +[x.y.z.w] +deep = 1 +[x] +top = 2 +[x.y] +mid = 3 +`)}) + require.NoError(t, err) + assert.Equal(t, map[string]any{"x": map[string]any{ + "top": 2.0, "y": map[string]any{"mid": 3.0, "z": map[string]any{"w": map[string]any{"deep": 1.0}}}, + }}, root.Value) +} + +func TestTOMLInfinityAndNaNAreReadOnly(t *testing.T) { + root, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte("a = inf\nb = -inf\nc = nan\nd = 1.5\n")}) + require.NoError(t, err) + for key, raw := range map[string]string{"a": "inf", "b": "-inf", "c": "nan"} { + assert.Equal(t, adapter.CapabilityReadOnly, root.Fields[key].Capability, key) + assert.Equal(t, raw, root.Fields[key].Value, key) + } + assert.Equal(t, 1.5, root.Fields["d"].Value) + assert.Equal(t, adapter.CapabilityDirect, root.Fields["d"].Capability) +} + +func TestTOMLClosedTablesCannotBeExtended(t *testing.T) { + for name, input := range map[string]string{ + "dotted key into inline table": "a = { x = 1 }\na.y = 2\n", + "header into inline table": "a = { x = 1 }\n[a.b]\ny = 2\n", + "dotted key into header table": "[a.b]\nx = 1\n[a]\nb.y = 2\n", + "dotted key into array table": "[[a]]\nx = 1\n[b]\n[c]\n", + } { + t.Run(name, func(t *testing.T) { + if name == "dotted key into array table" { + input = "[[a]]\nx = 1\n" // The root table is closed once a header opens. + _, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte("a.y = 1\n[[a]]\nx = 1\n")}) + require.ErrorContains(t, err, "incompatible definitions") + return + } + _, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte(input)}) + require.ErrorContains(t, err, "cannot be extended here") + }) + } + + root, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte(` +a.b.c = 1 +a.b.d = 2 +[a.e] +f = 3 +`)}) + require.NoError(t, err) + assert.Equal(t, map[string]any{"a": map[string]any{ + "b": map[string]any{"c": 1.0, "d": 2.0}, "e": map[string]any{"f": 3.0}, + }}, root.Value) +} + +func TestTOMLSubtablesOfArrayItemsAreReadOnly(t *testing.T) { + root, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte("[[servers]]\nname = \"one\"\n[servers.limits]\nmax = 1\n[servers.limits.deep]\nvalue = \"x\"\n[other]\nkept = true\n")}) + require.NoError(t, err) + item := root.Fields["servers"].Items[0] + limits := item.Fields["limits"] + assert.Equal(t, adapter.CapabilityReadOnly, limits.Capability) + assert.Equal(t, adapter.CapabilityReadOnly, limits.Fields["max"].Capability) + assert.Equal(t, adapter.CapabilityReadOnly, limits.Fields["deep"].Fields["value"].Capability) + assert.Equal(t, adapter.CapabilityDirect, root.Fields["other"].Fields["kept"].Capability) +} + +func TestTOMLTablesAreStructurallyReadOnlyWithWritableValues(t *testing.T) { + root, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte( + "[model]\nname = \"gpt\"\nlimits = { max = 1 }\n[model.parameters]\ntemperature = 0.2\n", + )}) + require.NoError(t, err) + model := root.Fields["model"] + for name, table := range map[string]*adapter.Node{ + "root": root, "header table": model, "sub-table": model.Fields["parameters"], "inline table": model.Fields["limits"], + } { + assert.Equal(t, adapter.CapabilityReadOnly, table.Capability, name) + assert.Equal(t, tomlStructureReason, table.Reason, name) + assert.Nil(t, table.Encode, name) + } + assert.Equal(t, adapter.CapabilityDirect, model.Fields["name"].Capability) + assert.Equal(t, adapter.CapabilityDirect, model.Fields["limits"].Fields["max"].Capability) + assert.Equal(t, adapter.CapabilityDirect, model.Fields["parameters"].Fields["temperature"].Capability) +} + +func TestTOMLNestedInlineTablesCannotBeExtended(t *testing.T) { + _, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte("a = { b = { x = 1 }, b.y = 2 }\n")}) + require.ErrorContains(t, err, "cannot be extended here", "a dotted key cannot add to a nested inline table") + + root, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte("a = { b.x = 1, b.y = 2 }\n")}) + require.NoError(t, err) + assert.Equal(t, map[string]any{"a": map[string]any{"b": map[string]any{"x": 1.0, "y": 2.0}}}, root.Value) +} + +func TestTOMLImplicitHeaderTablesAreClosedToDottedKeys(t *testing.T) { + _, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte("[a.b.c]\nx = 1\n[a]\nb.y = 2\n")}) + require.ErrorContains(t, err, "cannot be extended here", "a table that a header path created cannot take a dotted key") + + root, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte("[a.b.c]\nx = 1\n[a.b]\ny = 2\n")}) + require.NoError(t, err, "a later header defines an implicit table once") + assert.Equal(t, map[string]any{"a": map[string]any{"b": map[string]any{"y": 2.0, "c": map[string]any{"x": 1.0}}}}, root.Value) +} + +func TestTOMLNumbersKeepTheirTypeAndPrecision(t *testing.T) { + root, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte("big = 9007199254740993\nratio = 1.0\ncount = 3\n")}) + require.NoError(t, err) + big := root.Fields["big"] + assert.Equal(t, adapter.CapabilityReadOnly, big.Capability, "an integer beyond 53 bits is read-only text") + assert.Equal(t, "9007199254740993", big.Value) + + huge, err := parseTOMLDocument(adapter.Document{Path: "value.toml", Data: []byte("huge = 99999999999999999999\nname = \"ok\"\n")}) + require.NoError(t, err, "an integer beyond int64 does not fail the document") + assert.Equal(t, adapter.CapabilityReadOnly, huge.Fields["huge"].Capability) + assert.Equal(t, "99999999999999999999", huge.Fields["huge"].Value) + assert.Equal(t, adapter.CapabilityDirect, huge.Fields["name"].Capability, "the other fields stay usable") + + encoded, err := root.Fields["ratio"].Encode(2.0) + require.NoError(t, err) + assert.Equal(t, "2.0", string(encoded), "a float slot keeps a float form") + encoded, err = root.Fields["count"].Encode(4.0) + require.NoError(t, err) + assert.Equal(t, "4", string(encoded), "an integer slot writes an integer") + _, err = root.Fields["count"].Encode(4.5) + require.Error(t, err, "an integer slot does not take a fraction") +} diff --git a/internal/sync/adapter/structured/yaml.go b/internal/sync/adapter/structured/yaml.go new file mode 100644 index 000000000..3f3616629 --- /dev/null +++ b/internal/sync/adapter/structured/yaml.go @@ -0,0 +1,75 @@ +package structured + +import ( + "bytes" + "io" + "unicode/utf8" + + "gopkg.in/yaml.v3" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" + "github.com/launchdarkly/ldcli/internal/sync/adapter/yamltree" +) + +type yamlAdapter struct{} + +func (yamlAdapter) Format() string { return "yaml" } +func (yamlAdapter) Behavior() string { return "yaml/1" } + +func (yamlAdapter) Inspect(bundle adapter.SourceBundle) (*adapter.Node, error) { + return parseYAMLDocument(bundle.Primary) +} + +func (yamlAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "$", nil } + +func init() { adapter.MustRegister(yamlAdapter{}) } + +func parseYAMLDocument(document adapter.Document) (*adapter.Node, error) { + if !utf8.Valid(document.Data) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "YAML source is not valid UTF-8", + "Save the file as UTF-8.", + ).WithFile(document.Path, 0, 0) + } + data := bytes.TrimPrefix(document.Data, []byte("\ufeff")) + offset := len(document.Data) - len(data) + decoder := yaml.NewDecoder(bytes.NewReader(data)) + var first yaml.Node + if err := decoder.Decode(&first); err != nil { + return nil, syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML source is invalid", "Fix the YAML syntax.", + ).WithFile(document.Path, 0, 0), err) + } + // The source has several documents when any later document has content. + // A trailing "---" decodes as an empty document, which is not content. + multiple := false + for !multiple { + var extra yaml.Node + extraErr := decoder.Decode(&extra) + if extraErr == io.EOF { + break + } + if extraErr != nil { + return nil, syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML source is invalid", "Fix the YAML syntax.", + ).WithFile(document.Path, 0, 0), extraErr) + } + multiple = len(extra.Content) != 0 && + !(extra.Content[0].Kind == yaml.ScalarNode && extra.Content[0].Tag == "!!null" && extra.Content[0].Value == "") + } + if len(first.Content) == 0 { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML document is empty", "Add one YAML value.", + ).WithFile(document.Path, 0, 0) + } + root, err := yamltree.Build(data, first.Content[0], offset, false) + if err != nil { + return nil, nameDocument(err, document.Path) + } + if multiple { + root.MarkReadOnly("multi-document YAML is read-only") + } + return root, nil +} diff --git a/internal/sync/adapter/yamltree/yamltree.go b/internal/sync/adapter/yamltree/yamltree.go index c4df24ffd..3ac926e47 100644 --- a/internal/sync/adapter/yamltree/yamltree.go +++ b/internal/sync/adapter/yamltree/yamltree.go @@ -103,14 +103,12 @@ func yamlPlainScalar(text []byte, flow bool) []byte { func quoteYAML(value, previous string) string { if strings.HasPrefix(previous, "'") { if strings.ContainsAny(value, "\r\n") { - encoded, _ := json.Marshal(value) - return string(encoded) + return quotedYAML(value) } return "'" + strings.ReplaceAll(value, "'", "''") + "'" } if strings.HasPrefix(previous, `"`) { - encoded, _ := json.Marshal(value) - return string(encoded) + return quotedYAML(value) } if value != "" && !strings.ContainsAny(value, ":\n\r#{}[],&*!|>'\"%@`") && value == strings.TrimSpace(value) { @@ -121,7 +119,17 @@ func quoteYAML(value, previous string) string { } } } - encoded, _ := json.Marshal(value) + return quotedYAML(value) +} + +// quotedYAML writes a double-quoted YAML string. JSON quoting is valid YAML, +// and the encoder keeps <, >, and & as they are. +func quotedYAML(value string) string { + encoded, err := adapter.EncodeJSON(value, "", "") + if err != nil { + fallback, _ := json.Marshal(value) + return string(fallback) + } return string(encoded) } @@ -296,14 +304,44 @@ func yamlValueNode(data []byte, node *yaml.Node, offset int, flow bool) (*adapte break } result = adapter.Bool(value, start, end, nil) - case "!!int", "!!float": + case "!!int": + // An integer beyond 53 bits has no exact float64, so it stays + // read-only text instead of a rounded number. + var integer int64 + if err := node.Decode(&integer); err != nil || !adapter.ExactInteger(integer) { + result = adapter.ReadOnly(start, end, "YAML integer is too large to represent exactly") + result.Value = node.Value + break + } + // An integer slot takes a whole number only, written as an integer, + // so that a write does not change the type of the value. + result = adapter.Number(float64(integer), start, end, func(next float64) ([]byte, error) { + if next != math.Trunc(next) || next > float64(adapter.MaxExactInteger) || next < -float64(adapter.MaxExactInteger) { + return nil, fmt.Errorf("YAML integer cannot hold %v", next) + } + return []byte(strconv.FormatInt(int64(next), 10)), nil + }) + case "!!float": + // YAML tags an integer beyond int64 as a float, so an integer + // literal here is too large as well. + if !strings.ContainsAny(node.Value, ".eE") && !strings.HasPrefix(node.Value, "0x") && !strings.HasPrefix(node.Value, "0o") { + result = adapter.ReadOnly(start, end, "YAML integer is too large to represent exactly") + result.Value = node.Value + break + } var number float64 if err := node.Decode(&number); err != nil || math.IsInf(number, 0) || math.IsNaN(number) { result = unreadableScalar(node, start, end) break } + // A float slot keeps a fraction or an exponent, so that a whole + // number does not turn the slot into an integer. result = adapter.Number(number, start, end, func(next float64) ([]byte, error) { - return []byte(strconv.FormatFloat(next, 'g', -1, 64)), nil + text := strconv.FormatFloat(next, 'g', -1, 64) + if !strings.ContainsAny(text, ".eE") { + text += ".0" + } + return []byte(text), nil }) case "!!null": result = adapter.ReadOnly(start, end, "YAML null values are read-only")