diff --git a/internal/sync/adapter/adapter.go b/internal/sync/adapter/adapter.go new file mode 100644 index 000000000..d41b749df --- /dev/null +++ b/internal/sync/adapter/adapter.go @@ -0,0 +1,171 @@ +// Package adapter is the contract between the sync engine and one source +// format. An adapter turns the bytes of a file into a tree of logical nodes. +// This package resolves select and map paths on that tree, builds typed +// projections, and plans exact edits. Adapters never read files, run code, or +// use the network, so they cannot get update planning, no-op detection, or +// capability checks wrong. +package adapter + +import ( + "fmt" + "sync" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// Adapter inspects one source format. +// +// Format is the name that a descriptor and the --format flag use. Behavior +// names the version of the mapping rules of the adapter, such as python/1. +// It changes when a change to the adapter can change a mapped value, so that +// every binding of the format gets a review. Inspect returns the logical tree +// of a source bundle. Defaults returns the select path and the field map that +// a descriptor of a kind can leave out, or an empty select path when the +// format needs an explicit one. +type Adapter interface { + Format() string + Behavior() string + Inspect(SourceBundle) (*Node, error) + Defaults(syncdomain.Kind) (selectPath string, fieldMap map[string]string) +} + +// DependencyReporter is an optional adapter interface. It lists the +// repository files, relative to the repository root, whose bytes can change +// the projection of a primary source. The host reads them. +type DependencyReporter interface { + Dependencies(Document) ([]string, error) +} + +// Document is one repository file supplied to an adapter. +type Document struct { + Path string + Data []byte +} + +// SourceBundle is a primary source and the dependencies that its format +// reported, by repository-relative path. +type SourceBundle struct { + Primary Document + Dependencies map[string]Document +} + +// Capability is the shared mapped-field capability. +type Capability = syncdomain.Capability + +const ( + CapabilityDirect = syncdomain.CapabilityDirect + CapabilityReadOnly = syncdomain.CapabilityReadOnly +) + +// TextEdit replaces one exact byte range of one file. +type TextEdit struct { + File string + Start int + End int + Expected []byte + Replacement []byte +} + +// FieldUpdate is the capability of one mapped field and the edits that give +// it the desired value. +type FieldUpdate struct { + Capability Capability + Edits []TextEdit + Reason string +} + +// UpdateResult has one entry for each mapped field whose value changes. +type UpdateResult struct { + Fields map[syncdomain.Field]FieldUpdate +} + +type registry struct { + mu sync.RWMutex + adapters map[string]Adapter +} + +var formats = registry{adapters: make(map[string]Adapter)} + +// Register adds one source format. A format name registers once. +func Register(registered Adapter) error { + if registered == nil || registered.Format() == "" { + return fmt.Errorf("adapter format is required") + } + formats.mu.Lock() + defer formats.mu.Unlock() + if _, exists := formats.adapters[registered.Format()]; exists { + return fmt.Errorf("source format %q is already registered", registered.Format()) + } + formats.adapters[registered.Format()] = registered + return nil +} + +// MustRegister registers a format from an init function. +func MustRegister(registered Adapter) { + if err := Register(registered); err != nil { + panic(err) + } +} + +func lookup(format string) (Adapter, error) { + formats.mu.RLock() + registered := formats.adapters[format] + formats.mu.RUnlock() + if registered == nil { + return nil, syncdomain.NewProblem( + syncdomain.ErrorUnsupportedFormat, + fmt.Sprintf("source format %q is not supported", format), + "Use a registered format name.", + ) + } + return registered, nil +} + +// Behavior returns the behavior version of a registered format. +func Behavior(format string) (string, error) { + registered, err := lookup(format) + if err != nil { + return "", err + } + return registered.Behavior(), nil +} + +// DependencyPaths returns the repository files that a format reports for a +// primary source. +func DependencyPaths(format string, primary Document) ([]string, error) { + registered, err := lookup(format) + if err != nil { + return nil, err + } + reporter, ok := registered.(DependencyReporter) + if !ok { + return nil, nil + } + return reporter.Dependencies(primary) +} + +// Parse returns the logical tree of a bundle in the named format. Every node +// without a file gets the path of the primary document. +func Parse(bundle SourceBundle, format string) (*Node, error) { + registered, err := lookup(format) + if err != nil { + return nil, err + } + root, err := registered.Inspect(bundle) + if err != nil { + if _, ok := syncdomain.ProblemFrom(err); !ok { + err = syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + err.Error(), + "Fix the source syntax or use a supported static form.", + ).WithFile(bundle.Primary.Path, 0, 0), err) + } + } + root.SetFile(bundle.Primary.Path) + return root, err +} + +// ParseDocument returns the logical tree of one document without dependencies. +func ParseDocument(document Document, format string) (*Node, error) { + return Parse(SourceBundle{Primary: document}, format) +} diff --git a/internal/sync/adapter/adapter_test.go b/internal/sync/adapter/adapter_test.go new file mode 100644 index 000000000..7274b3132 --- /dev/null +++ b/internal/sync/adapter/adapter_test.go @@ -0,0 +1,496 @@ +package adapter + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestPathParserRejectsEveryUnsupportedShape(t *testing.T) { + paths := []string{ + `[id="one"]`, + `$["key"]`, + `items[id`, + `items[id!`, + `items[id="one"][name="two"]`, + `items[id="one",name=two]`, + `items[id="one",name]`, + `items[id="one",="two"]`, + `items[id="one",name="two"`, + `items[id="\uD800"]`, + `items[id="\uDC00"]`, + `items[id="\uD800\u0041"]`, + `items[id="bad\q"]`, + `["unclosed]`, + `items[id="one"] trailing`, + } + for _, path := range paths { + t.Run(path, func(t *testing.T) { + err := ParsePath(path) + require.Error(t, err) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorInvalidPathSyntax, problem.Code) + }) + } + require.NoError(t, ParsePath(`items[id="\uD83D\uDE00"]`)) + assert.Error(t, pathSyntaxError("x", 10, "test")) +} + +func TestSourcePathFailuresUseStableCodes(t *testing.T) { + root := &Node{Fields: map[string]*Node{ + "scalar": {Value: "text"}, + "items": {Items: []*Node{ + {Fields: map[string]*Node{"id": {Value: 10}}}, + }}, + }} + tests := []struct { + name, path string + code syncdomain.ErrorCode + }{ + {name: "missing field", path: "missing", code: syncdomain.ErrorPathNotFound}, + {name: "field on scalar", path: "scalar.child", code: syncdomain.ErrorPathTypeMismatch}, + {name: "filter type", path: `items[id="10"]`, code: syncdomain.ErrorFilterTypeMismatch}, + {name: "no filter match", path: `items[name="missing"]`, code: syncdomain.ErrorPathNotFound}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := root.Resolve(test.path) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, test.code, problem.Code) + }) + } +} + +func TestBuildProjectionAcceptsAllMappedFieldTypes(t *testing.T) { + selected := &Node{Fields: map[string]*Node{ + "prompt": {Value: "Help"}, + "messages": {Value: []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "Help"}}}, + "name": {Value: "assistant"}, + "model": {Value: "model"}, + "temperature": {Value: int64(1)}, + }} + fieldMap := map[string]string{ + string(syncdomain.FieldPrompt): "prompt", + string(syncdomain.FieldName): "name", + string(syncdomain.FieldModel): "model", + `model.parameters["temperature"]`: "temperature", + } + projection, err := buildProjection(syncdomain.KindVariation, selected, fieldMap) + require.NoError(t, err) + variation := projection.(syncdomain.VariationProjection) + assert.Equal(t, syncdomain.VariationModeAgent, variation.Mode) + assert.Equal(t, 1.0, (*variation.Parameters)["temperature"]) + + projection, err = buildProjection( + syncdomain.KindVariation, selected, + map[string]string{string(syncdomain.FieldMessages): "messages"}, + ) + require.NoError(t, err) + assert.NotNil(t, projection.(syncdomain.VariationProjection).Messages) + + _, err = buildProjection(syncdomain.KindVariation, selected, map[string]string{ + string(syncdomain.FieldPrompt): "prompt", string(syncdomain.FieldMessages): "messages", + }) + require.ErrorContains(t, err, "cannot both be set") + _, err = buildProjection(syncdomain.KindVariation, selected, map[string]string{string(syncdomain.FieldName): "name"}) + require.ErrorContains(t, err, "is required for a variation") + + selected = &Node{Fields: map[string]*Node{ + "description": {Value: "Search"}, + "schema": {Value: map[string]any{"type": "object"}}, + "parameters": {Value: map[string]any{"limit": 10}}, + "tags": {Value: []any{"search", "safe"}}, + }} + projection, err = buildProjection(syncdomain.KindTool, selected, map[string]string{ + string(syncdomain.FieldDescription): "description", + string(syncdomain.FieldSchema): "schema", + string(syncdomain.FieldCustomParameters): "parameters", + string(syncdomain.FieldTags): "tags", + }) + require.NoError(t, err) + assert.Equal(t, []string{"search", "safe"}, *projection.(syncdomain.ToolProjection).Tags) + + selected = &Node{Fields: map[string]*Node{ + "description": {Value: "Review"}, + "markdown": {Value: "# Review"}, + }} + projection, err = buildProjection(syncdomain.KindSkill, selected, map[string]string{ + string(syncdomain.FieldDescription): "description", + string(syncdomain.FieldMarkdown): "markdown", + }) + require.NoError(t, err) + assert.Equal(t, "# Review", *projection.(syncdomain.SkillProjection).Markdown) +} + +func TestBuildProjectionRejectsInvalidMappedFieldTypes(t *testing.T) { + tests := []struct { + name string + kind syncdomain.Kind + mode syncdomain.VariationMode + field syncdomain.Field + value any + }{ + {name: "agent prompt", kind: syncdomain.KindVariation, field: syncdomain.FieldPrompt, value: true}, + {name: "messages", kind: syncdomain.KindVariation, mode: syncdomain.VariationModeCompletion, field: syncdomain.FieldMessages, value: true}, + {name: "message role", kind: syncdomain.KindVariation, mode: syncdomain.VariationModeCompletion, field: syncdomain.FieldMessages, value: []syncdomain.Message{{Role: "invalid"}}}, + {name: "name", kind: syncdomain.KindVariation, field: syncdomain.FieldName, value: true}, + {name: "model", kind: syncdomain.KindVariation, field: syncdomain.FieldModel, value: true}, + {name: "parameters", kind: syncdomain.KindVariation, field: syncdomain.FieldModelParameters, value: "hot"}, + {name: "description", kind: syncdomain.KindTool, field: syncdomain.FieldDescription, value: true}, + {name: "schema", kind: syncdomain.KindTool, field: syncdomain.FieldSchema, value: true}, + {name: "parameters", kind: syncdomain.KindTool, field: syncdomain.FieldCustomParameters, value: true}, + {name: "tags", kind: syncdomain.KindTool, field: syncdomain.FieldTags, value: []any{"ok", 1}}, + {name: "skill", kind: syncdomain.KindSkill, field: syncdomain.FieldMarkdown, value: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + selected := &Node{Fields: map[string]*Node{ + string(test.field): {Value: test.value}, + }} + fieldMap := map[string]string{string(test.field): string(test.field)} + if test.kind == syncdomain.KindVariation && test.field != syncdomain.FieldPrompt && test.field != syncdomain.FieldMessages { + selected.Fields["prompt"] = &Node{Value: "Help"} + fieldMap[string(syncdomain.FieldPrompt)] = "prompt" + } + _, err := buildProjection(test.kind, selected, fieldMap) + require.Error(t, err) + }) + } + + _, err := buildProjection(syncdomain.KindVariation, &Node{}, map[string]string{}) + require.Error(t, err) + _, err = buildProjection(syncdomain.Kind("unknown"), &Node{}, nil) + require.Error(t, err) +} + +func TestProjectionFieldReadsEveryOptionalField(t *testing.T) { + text := "value" + list := []string{"one"} + messages := []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "Help"}} + object := map[string]any{"type": "object"} + parameters := map[string]any{"temperature": 0.5} + projections := []syncdomain.Projection{ + syncdomain.VariationProjection{ + Instructions: &text, Name: &text, Model: &text, Parameters: ¶meters, Tools: &list, Skills: &list, + }, + syncdomain.VariationProjection{Messages: &messages}, + syncdomain.ToolProjection{ + Description: &text, Schema: &object, CustomParameters: &object, Tags: &list, + }, + syncdomain.SkillProjection{Description: &text, Markdown: &text}, + } + fields := [][]syncdomain.Field{ + {syncdomain.FieldPrompt, syncdomain.FieldName, syncdomain.FieldModel, syncdomain.FieldModelParameters, syncdomain.FieldTools, syncdomain.FieldSkills}, + {syncdomain.FieldMessages}, + {syncdomain.FieldDescription, syncdomain.FieldSchema, syncdomain.FieldCustomParameters, syncdomain.FieldTags}, + {syncdomain.FieldDescription, syncdomain.FieldMarkdown}, + } + for index, projection := range projections { + for _, field := range fields[index] { + _, present := projection.Value(field) + assert.True(t, present) + } + } +} + +func TestMessageUpdateSafetyBranches(t *testing.T) { + have := []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "old"}} + tests := []struct { + name string + node *Node + want []syncdomain.Message + }{ + {name: "different count", node: &Node{}, want: nil}, + {name: "different role", node: &Node{}, want: []syncdomain.Message{{Role: syncdomain.RoleUser, Content: "old"}}}, + {name: "missing content", node: &Node{}, want: []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "new"}}}, + {name: "read only content", node: &Node{Items: []*Node{{Fields: map[string]*Node{"content": {Reason: "dynamic"}}}}}, want: []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "new"}}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + update := updateMessageContents(SourceBundle{}, test.node, have, test.want) + assert.Equal(t, CapabilityReadOnly, update.Capability) + }) + } + + content := String("old", 0, 5, func(string) ([]byte, error) { return []byte(`"new"`), nil }) + content.File = "missing.py" + update := updateMessageContents( + SourceBundle{Primary: Document{Path: "main.py"}}, + &Node{Items: []*Node{{Fields: map[string]*Node{"content": content}}}}, + have, []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "new"}}, + ) + assert.Equal(t, CapabilityReadOnly, update.Capability) +} + +func TestSourceValueHelpersRejectDynamicValues(t *testing.T) { + assert.False(t, semanticEqual(make(chan int), make(chan int))) + _, ok := AsNumber(math.NaN()) + assert.True(t, ok) + _, ok = AsNumber("1") + assert.False(t, ok) + _, ok = stringList(true) + assert.False(t, ok) + _, ok = objectValue(true) + assert.False(t, ok) + + stringValue := String("old", 0, 3, func(value string) ([]byte, error) { return []byte(value), nil }) + _, err := stringValue.Encode(1) + require.Error(t, err) + numberValue := Number(1, 0, 1, func(float64) ([]byte, error) { return []byte("1"), nil }) + _, err = numberValue.Encode("one") + require.Error(t, err) +} + +func TestRegistryContractAndDependencyErrors(t *testing.T) { + require.Error(t, Register(nil)) + require.Error(t, Register(contractAdapter{})) + require.NoError(t, Register(namedFakeAdapter{format: "contract-test"})) + require.Error(t, Register(namedFakeAdapter{format: "contract-test"})) + + _, err := lookup("missing-format") + require.Error(t, err) + dependencies, err := DependencyPaths("contract-test", Document{}) + require.NoError(t, err) + assert.Nil(t, dependencies) + _, err = DependencyPaths("missing-format", Document{}) + require.Error(t, err) +} + +func TestParsePathGrammar(t *testing.T) { + for _, path := range []string{ + "$", + "field.child", + `items[id="value"]`, + `items[role="system",name="intro"]`, + "items[0]", + "items[0][1]", + "items[2147483647]", + `items[id="value"].messages[0].content`, + `object["literal.key"]`, + `["top level"]`, + } { + require.NoError(t, ParsePath(path), path) + } + for _, path := range []string{ + "", + "$.field", + "items[]", + "items[-1]", + "items[1.0]", + "items[01]", + "items[2147483648]", + `items[0][id="value"]`, + "items[*]", + `items[id=value]`, + `items[ id = "value" ]`, + "items..id", + } { + err := ParsePath(path) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok, path) + assert.Equal(t, syncdomain.ErrorInvalidPathSyntax, problem.Code, path) + } +} + +func TestResolvePathSupportsChainedAndMixedIndexes(t *testing.T) { + root := &Node{Fields: map[string]*Node{ + "agents": {Items: []*Node{ + {Fields: map[string]*Node{ + "id": {Value: "support"}, + "messages": {Items: []*Node{ + {Fields: map[string]*Node{"content": {Value: "First"}}}, + {Fields: map[string]*Node{"content": {Value: "Second"}}}, + }, Collection: true}, + }}, + }, Collection: true}, + }} + + value, err := root.Resolve(`agents[id="support"].messages[1].content`) + + require.NoError(t, err) + assert.Equal(t, "Second", value.Value) +} + +func TestResolvePathIndexErrorsUseStableCodes(t *testing.T) { + root := &Node{Fields: map[string]*Node{ + "items": {Items: []*Node{{Value: "only"}}, Collection: true}, + "value": {Value: "text"}, + }} + tests := []struct { + path string + code syncdomain.ErrorCode + }{ + {path: "items[1]", code: syncdomain.ErrorIndexOutOfRange}, + {path: "value[0]", code: syncdomain.ErrorPathTypeMismatch}, + } + for _, test := range tests { + _, err := root.Resolve(test.path) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, test.code, problem.Code) + } +} + +func TestResolvePathSupportsLiteralKeysAndRejectsAmbiguousSelectors(t *testing.T) { + root := &Node{Fields: map[string]*Node{ + "metadata": {Fields: map[string]*Node{"display.name": {Value: "Support"}}}, + "agents": { + Items: []*Node{ + {Fields: map[string]*Node{"id": {Value: "support"}}}, + {Fields: map[string]*Node{"id": {Value: "support"}}}, + }, + }, + }} + value, err := root.Resolve(`metadata["display.name"]`) + require.NoError(t, err) + assert.Equal(t, "Support", value.Value) + + _, err = root.Resolve(`agents[id="support"]`) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorPathAmbiguous, problem.Code) +} + +func TestResolvePathMatchesEveryFilterCondition(t *testing.T) { + root := &Node{Fields: map[string]*Node{ + "items": {Items: []*Node{ + {Fields: map[string]*Node{ + "role": {Value: "system"}, "name": {Value: "intro"}, "content": {Value: "First"}, + }}, + {Fields: map[string]*Node{ + "role": {Value: "system"}, "name": {Value: "outro"}, "content": {Value: "Second"}, + }}, + }}, + }} + + value, err := root.Resolve(`items[role="system",name="intro"].content`) + + require.NoError(t, err) + assert.Equal(t, "First", value.Value) +} + +func TestRegisteredAdapterRunsThroughSharedCore(t *testing.T) { + require.NoError(t, Register(fakeAdapter{})) + id := syncdomain.VariationID("project", "config", "default") + + projection, err := ParseSource( + bundle("prompt.fake", []byte("Registered without a core change.")), + id, syncdomain.Reference{File: "prompt.fake", Format: "fake-test"}, + ) + + require.NoError(t, err) + assert.Equal(t, "Registered without a core change.", *projection.(syncdomain.VariationProjection).Instructions) +} + +func TestRegistryRejectsInvalidAndDuplicateAdapters(t *testing.T) { + require.Error(t, Register(nil)) + adapter := namedFakeAdapter{format: "duplicate-test"} + require.NoError(t, Register(adapter)) + require.Error(t, Register(adapter)) + + _, err := lookup("not-registered") + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorUnsupportedFormat, problem.Code) +} + +func TestIndexedSelectionIdentityUsesStableFieldsAndFallback(t *testing.T) { + identified := &Node{Fields: map[string]*Node{ + "id": {Value: "support"}, + "n": {Value: 1}, + }} + assert.Equal(t, map[string]any{"id": "support"}, directSelectionIdentity(identified)) + assert.Nil(t, directSelectionIdentity(nil)) + assert.True(t, scalarValue(uint64(2))) + assert.False(t, scalarValue(nil)) + assert.False(t, (*Node)(nil).IsCollection()) + assert.True(t, (&Node{Value: []string{"one"}}).IsCollection()) + assert.Equal(t, + []any{"one"}, + stableNodeValue(&Node{Items: []*Node{{Value: "one"}}, Collection: true}, map[*Node]bool{}), + ) + assert.Nil(t, stableNodeValue(&Node{}, map[*Node]bool{})) + + cycle := &Node{Fields: map[string]*Node{}} + cycle.Fields["self"] = cycle + fallback := &Node{Fields: map[string]*Node{"prompt": {Value: "Help"}}} + selections, err := indexedSelections( + []indexedResolution{ + {path: "items[0]", collectionLength: 1, item: identified}, + {path: "items[0].children[0]", collectionLength: 1, item: fallback}, + {path: "items[0].cycle[0]", collectionLength: 1, item: cycle}, + }, + fallback, + ) + require.NoError(t, err) + require.Len(t, selections, 3) + for _, selection := range selections { + assert.NotEmpty(t, selection.Fingerprint) + } + + _, err = indexedSelections( + []indexedResolution{{path: "items[0]", collectionLength: 1, item: &Node{Value: make(chan int)}}}, + fallback, + ) + require.ErrorContains(t, err, "encode indexed selection fingerprint") +} + +type fakeAdapter struct{} + +func (fakeAdapter) Format() string { return "fake-test" } +func (fakeAdapter) Behavior() string { return "fake-test/1" } +func (fakeAdapter) Inspect(bundle SourceBundle) (*Node, error) { + document := bundle.Primary + return String(string(document.Data), 0, len(document.Data), func(text string) ([]byte, error) { + return []byte(text), nil + }), nil +} +func (fakeAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + return "$", map[string]string{string(syncdomain.FieldPrompt): "$"} +} + +type namedFakeAdapter struct{ format string } + +func (adapter namedFakeAdapter) Format() string { return adapter.format } +func (namedFakeAdapter) Behavior() string { return "fake-test/1" } +func (namedFakeAdapter) Inspect(bundle SourceBundle) (*Node, error) { + return fakeAdapter{}.Inspect(bundle) +} +func (namedFakeAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + return fakeAdapter{}.Defaults(kind) +} + +type contractAdapter struct{} + +func (contractAdapter) Format() string { return "" } +func (contractAdapter) Behavior() string { return "" } +func (contractAdapter) Inspect(SourceBundle) (*Node, error) { return nil, nil } +func (contractAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "$", nil } + +func bundle(path string, data []byte) SourceBundle { + return SourceBundle{Primary: Document{Path: path, Data: data}} +} + +func TestCoreHelpersRejectUnknownKindsAndFailedEncoders(t *testing.T) { + assert.False(t, fieldAllowed(syncdomain.Kind("unknown"), syncdomain.FieldPrompt)) + _, err := ParseDocument(Document{}, "missing") + require.Error(t, err) + number, ok := AsNumber(1) + assert.True(t, ok) + assert.Equal(t, 1.0, number) + + content := String("old", 0, 3, func(string) ([]byte, error) { return nil, assert.AnError }) + update := updateMessageContents( + bundle("value.json", []byte("old")), + &Node{Items: []*Node{{Fields: map[string]*Node{"content": content}}}}, + []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "old"}}, + []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "new"}}, + ) + assert.Equal(t, CapabilityReadOnly, update.Capability) +} diff --git a/internal/sync/adapter/markdown/markdown.go b/internal/sync/adapter/markdown/markdown.go new file mode 100644 index 000000000..0ee5e91e0 --- /dev/null +++ b/internal/sync/adapter/markdown/markdown.go @@ -0,0 +1,363 @@ +// Package markdown inspects Markdown documents and Agent Skill files. +// +// A Markdown document exposes four logical fields. frontmatter is the YAML +// front matter, or an empty object. body is the text after the front matter. +// sections has one item for each heading, with the heading text and the body +// below it. markers has one item for each LaunchDarkly marker region. The +// same adapter serves plain prompt files, AGENTS.md, CLAUDE.md, Cursor rules, +// and Copilot instructions, because only the mapped value changes. +package markdown + +import ( + "bytes" + "errors" + "fmt" + "strings" + "unicode/utf8" + + "gopkg.in/yaml.v3" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" + "github.com/launchdarkly/ldcli/internal/sync/adapter/yamltree" +) + +type markdownAdapter struct{} + +func (markdownAdapter) Format() string { return "markdown" } +func (markdownAdapter) Behavior() string { return "markdown/1" } + +func (markdownAdapter) Inspect(bundle adapter.SourceBundle) (*adapter.Node, error) { + return parse(bundle.Primary, false) +} + +func (markdownAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + if kind == syncdomain.KindVariation { + return "$", map[string]string{string(syncdomain.FieldPrompt): "body"} + } + return "$", nil +} + +type agentSkillAdapter struct{} + +func (agentSkillAdapter) Format() string { return "agent-skill" } +func (agentSkillAdapter) Behavior() string { return "agent-skill/1" } + +func (agentSkillAdapter) Inspect(bundle adapter.SourceBundle) (*adapter.Node, error) { + return parseAgentSkill(bundle.Primary) +} + +func (agentSkillAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + if kind == syncdomain.KindSkill { + return "$", map[string]string{ + string(syncdomain.FieldDescription): "frontmatter.description", + string(syncdomain.FieldMarkdown): "body", + } + } + return "$", nil +} + +func init() { + adapter.MustRegister(markdownAdapter{}) + adapter.MustRegister(agentSkillAdapter{}) +} + +// parse builds the logical document. With exactBody, the body keeps every +// byte, as a skill needs. Otherwise the body is normalized prompt text. +func parse(document adapter.Document, exactBody bool) (*adapter.Node, error) { + data := document.Data + if !utf8.Valid(data) { + return nil, fmt.Errorf("%s: source is not valid UTF-8", document.Path) + } + bodyStart := len(data) - len(bytes.TrimPrefix(data, []byte("\ufeff"))) + frontMatter := &adapter.Node{Value: map[string]any{}, Fields: map[string]*adapter.Node{}} + if hasFrontMatter(data[bodyStart:]) { + region, end, err := splitFrontMatter(data) + if err != nil { + return nil, fmt.Errorf("%s: %w", document.Path, err) + } + if frontMatter, err = parseFrontMatter(region); err != nil { + return nil, fmt.Errorf("%s: %w", document.Path, err) + } + bodyStart = end + } + body := data[bodyStart:] + bodyNode := textNode(body, bodyStart, len(data)) + if exactBody { + bodyNode = exactNode(body, bodyStart, len(data)) + } + markers, err := markerRegions(body, bodyStart) + if err != nil { + return nil, fmt.Errorf("%s: %w", document.Path, err) + } + return &adapter.Node{Fields: map[string]*adapter.Node{ + "frontmatter": frontMatter, + "body": bodyNode, + "sections": {Items: sections(body, bodyStart), Collection: true}, + "markers": markers, + }}, nil +} + +// parseAgentSkill builds a SKILL.md document and checks the fields that a +// skill needs: a string description in the front matter and a body. +func parseAgentSkill(document adapter.Document) (*adapter.Node, error) { + if !hasFrontMatter(bytes.TrimPrefix(document.Data, []byte("\ufeff"))) { + return nil, fmt.Errorf("%s: missing YAML front matter", document.Path) + } + root, err := parse(document, true) + if err != nil { + return nil, err + } + description := root.Fields["frontmatter"].Fields["description"] + switch { + case root.Fields["frontmatter"].Fields == nil: + return nil, fmt.Errorf("%s: front matter must be a mapping", document.Path) + case description == nil: + return nil, syncdomain.NewProblem( + syncdomain.ErrorPathNotFound, + "front matter field \"description\" was not found", + "Add the field or update the map path.", + ).WithFile(document.Path, 0, 0) + case description.Value == nil: + return nil, fmt.Errorf("%s: front matter description must be a string", document.Path) + } + if _, ok := description.Value.(string); !ok { + return nil, fmt.Errorf("%s: front matter description must be a string", document.Path) + } + if strings.TrimSpace(root.Fields["body"].Value.(string)) == "" { + return nil, fmt.Errorf("%s: skill Markdown is required", document.Path) + } + return root, nil +} + +// hasFrontMatter reports whether data starts with a front matter fence. A +// fence is three dashes at the start of a line and nothing else. +func hasFrontMatter(data []byte) bool { + return isFence(string(data[:nextLineEnd(data, 0)])) +} + +func isFence(line string) bool { + return strings.TrimRight(line, "\r\n") == "---" +} + +// splitFrontMatter returns the YAML region between the front matter fences +// and the offset where the body starts. +func splitFrontMatter(data []byte) (yamltree.Region, int, error) { + offset := len(data) - len(bytes.TrimPrefix(data, []byte("\ufeff"))) + contentStart := nextLineEnd(data, offset) + for lineStart := contentStart; lineStart < len(data); { + lineEnd := nextLineEnd(data, lineStart) + if isFence(string(data[lineStart:lineEnd])) { + bodyStart := lineEnd + if bytes.HasPrefix(data[bodyStart:], []byte("\r\n")) { + bodyStart += 2 + } else if bytes.HasPrefix(data[bodyStart:], []byte("\n")) { + bodyStart++ + } + return yamltree.Region{Data: data[contentStart:lineStart], StartOffset: contentStart}, bodyStart, nil + } + lineStart = lineEnd + } + return yamltree.Region{}, 0, errors.New("unclosed YAML front matter") +} + +// parseFrontMatter builds the logical tree of the front matter. Empty front +// matter is an empty object. +func parseFrontMatter(region yamltree.Region) (*adapter.Node, error) { + var document yaml.Node + if err := yaml.Unmarshal(region.Data, &document); err != nil { + return nil, fmt.Errorf("invalid front matter: %w", err) + } + if len(document.Content) == 0 { + return &adapter.Node{Value: map[string]any{}, Fields: map[string]*adapter.Node{}}, nil + } + return yamltree.Build(region.Data, document.Content[0], region.StartOffset, false) +} + +func nextLineEnd(data []byte, start int) int { + if index := bytes.IndexByte(data[start:], '\n'); index >= 0 { + return start + index + 1 + } + return len(data) +} + +// sections returns one node for each heading. The body of a section runs to +// the next heading of the same or a higher level. Headings inside fenced code +// and indented lines do not count. base is the offset of data in the file. +func sections(data []byte, base int) []*adapter.Node { + type heading struct { + level int + text string + lineStart int + bodyStart int + } + var headings []heading + forEachLine(data, func(line markdownLine) { + if line.fenced || line.indented { + return + } + trimmed := strings.TrimLeft(line.text, " ") + level := 0 + for level < len(trimmed) && trimmed[level] == '#' { + level++ + } + if level > 0 && level <= 6 && level < len(trimmed) && trimmed[level] == ' ' { + headings = append(headings, heading{ + level: level, text: headingText(trimmed[level:]), lineStart: line.start, bodyStart: line.end, + }) + } + }) + + result := make([]*adapter.Node, 0, len(headings)) + for index, current := range headings { + bodyEnd := len(data) + for next := index + 1; next < len(headings); next++ { + if headings[next].level <= current.level { + bodyEnd = headings[next].lineStart + break + } + } + result = append(result, &adapter.Node{Fields: map[string]*adapter.Node{ + "heading": {Value: current.text}, + "body": textNode(data[current.bodyStart:bodyEnd], base+current.bodyStart, base+bodyEnd), + }}) + } + return result +} + +// markdownLine is one line of a document with its byte range, whether it is +// inside or part of a fenced code block, and whether it is indented code. +type markdownLine struct { + start, end int + text string + fenced bool + indented bool +} + +// forEachLine visits each line of data in order and tracks fenced code +// blocks, so that headings and markers inside code do not count. As in +// CommonMark, a closing fence has nothing but spaces after it, and the info +// string of a backtick fence has no backtick. +func forEachLine(data []byte, visit func(markdownLine)) { + var fenceCharacter byte + fenceLength := 0 + for offset := 0; offset < len(data); { + end := nextLineEnd(data, offset) + text := strings.TrimRight(string(data[offset:end]), "\r\n") + trimmed := strings.TrimLeft(text, " ") + line := markdownLine{start: offset, end: end, text: text, fenced: fenceLength != 0} + line.indented = len(text)-len(trimmed) > 3 || strings.HasPrefix(trimmed, "\t") + if !line.indented { + marker, markerLength := fence(trimmed) + rest := trimmed[markerLength:] + switch { + case fenceLength == 0 && markerLength >= 3 && (marker == '~' || !strings.Contains(rest, "`")): + fenceCharacter, fenceLength, line.fenced = marker, markerLength, true + case fenceLength != 0 && marker == fenceCharacter && markerLength >= fenceLength && strings.TrimSpace(rest) == "": + fenceCharacter, fenceLength = 0, 0 + } + } + visit(line) + offset = end + } +} + +func fence(line string) (byte, int) { + if line == "" || line[0] != '`' && line[0] != '~' { + return 0, 0 + } + length := 1 + for length < len(line) && line[length] == line[0] { + length++ + } + return line[0], length +} + +// headingText returns the heading without its closing hashes. +func headingText(text string) string { + text = strings.TrimSpace(text) + lastSpace := strings.LastIndexByte(text, ' ') + if lastSpace < 0 || lastSpace == len(text)-1 || strings.Trim(text[lastSpace+1:], "#") != "" { + return text + } + return strings.TrimSpace(text[:lastSpace]) +} + +// markerRegions returns one node for each region between a begin and an end +// marker. A marker inside fenced or indented code does not count. Nested, +// reversed, and unclosed markers are errors. +func markerRegions(data []byte, base int) (*adapter.Node, error) { + const beginPrefix = ""): + if openID != "" { + err = fmt.Errorf("nested LaunchDarkly marker %q", openID) + return + } + openID = strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(text, beginPrefix), "-->")) + bodyStart = line.end + case strings.HasPrefix(text, endPrefix) && strings.HasSuffix(text, "-->"): + id := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(text, endPrefix), "-->")) + if openID == "" || id != openID { + err = fmt.Errorf("LaunchDarkly marker %q has no matching begin marker", id) + return + } + items = append(items, &adapter.Node{Fields: map[string]*adapter.Node{ + "id": {Value: id}, + "body": textNode(data[bodyStart:line.start], base+bodyStart, base+line.start), + }}) + openID = "" + } + }) + if err != nil { + return nil, err + } + if openID != "" { + return nil, fmt.Errorf("LaunchDarkly marker %q is not closed", openID) + } + return &adapter.Node{Items: items, Collection: true}, nil +} + +// textNode is prompt text: normalized line endings and no surrounding white +// space. An update keeps the white space around the text and the line ending +// style of the file. +func textNode(data []byte, start, end int) *adapter.Node { + lineEnding := "\n" + if bytes.Contains(data, []byte("\r\n")) { + lineEnding = "\r\n" + } + trimmed := bytes.TrimFunc(data, isSpace) + prefix := bytes.Clone(data[:bytes.Index(data, trimmed)]) + suffix := bytes.Clone(data[len(prefix)+len(trimmed):]) + return adapter.String(normalize(string(data)), start, end, func(text string) ([]byte, error) { + text = strings.ReplaceAll(normalize(text), "\n", lineEnding) + return bytes.Join([][]byte{prefix, []byte(text), suffix}, nil), nil + }) +} + +// exactNode is text that keeps every byte, as skill Markdown does. An update +// keeps the final newline of the file. +func exactNode(data []byte, start, end int) *adapter.Node { + finalNewline := bytes.HasSuffix(data, []byte("\n")) + return adapter.String(string(data), start, end, func(text string) ([]byte, error) { + if finalNewline && text != "" && !strings.HasSuffix(text, "\n") { + text += "\n" + } + return []byte(text), nil + }) +} + +func normalize(text string) string { + return syncdomain.NormalizePromptText(text) +} + +func isSpace(character rune) bool { + return character == ' ' || character == '\t' || character == '\r' || character == '\n' +} diff --git a/internal/sync/adapter/markdown/markdown_test.go b/internal/sync/adapter/markdown/markdown_test.go new file mode 100644 index 000000000..5af6b029b --- /dev/null +++ b/internal/sync/adapter/markdown/markdown_test.go @@ -0,0 +1,128 @@ +package markdown + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +func TestMarkdownRejectsUnsafeMarkersAndFrontMatter(t *testing.T) { + inputs := []string{ + "\n", + "", + "", + "---\ninvalid: [\n---\nbody", + "---\ndescription: value", + } + for _, input := range inputs { + _, err := parse(adapter.Document{Path: "AGENTS.md", Data: []byte(input)}, false) + require.Error(t, err, input) + } + + root, err := parse(adapter.Document{Path: "AGENTS.md", Data: []byte("\ufeff# Instructions")}, false) + require.NoError(t, err) + assert.Equal(t, "# Instructions", root.Fields["body"].Value) + assert.Empty(t, root.Fields["frontmatter"].Fields) + assert.Empty(t, root.Fields["markers"].Items) +} + +func TestMarkdownAndAgentSkillSafetyForms(t *testing.T) { + selectPath, fieldMap := markdownAdapter{}.Defaults(syncdomain.KindTool) + assert.Equal(t, "$", selectPath) + assert.Nil(t, fieldMap) + selectPath, fieldMap = agentSkillAdapter{}.Defaults(syncdomain.KindVariation) + assert.Equal(t, "$", selectPath) + assert.Nil(t, fieldMap) + + _, err := parse(adapter.Document{Path: "README.md", Data: []byte{0xff}}, false) + require.Error(t, err) + _, err = parseAgentSkill(adapter.Document{Path: "SKILL.md", Data: []byte{0xff}}) + require.Error(t, err) + + for name, input := range map[string]string{ + "missing front matter": "# Body", + "unclosed front matter": "---\ndescription: value", + "invalid YAML": "---\ndescription: [\n---\nBody", + "front matter scalar": "---\nvalue\n---\nBody", + "missing description": "---\nname: skill\n---\nBody", + "description type": "---\ndescription: [value]\n---\nBody", + "null description": "---\ndescription: null\n---\nBody", + "tilde description": "---\ndescription: ~\n---\nBody", + "blank description": "---\ndescription:\n---\nBody", + "empty body": "---\ndescription: value\n---\n", + } { + t.Run(name, func(t *testing.T) { + _, err := parseAgentSkill(adapter.Document{Path: "SKILL.md", Data: []byte(input)}) + require.Error(t, err) + }) + } + + root, err := parseAgentSkill(adapter.Document{Path: "SKILL.md", Data: []byte("---\ndescription: |\n long text\n---\nBody\n")}) + require.NoError(t, err) + assert.Equal(t, adapter.CapabilityReadOnly, root.Fields["frontmatter"].Fields["description"].Capability) + + headings := sections([]byte(" # indented\n````\n# hidden\n````\n## Visible ##\nbody\n"), 0) + require.Len(t, headings, 1) + assert.Equal(t, "Visible", headings[0].Fields["heading"].Value) + + exact := exactNode([]byte("body\n"), 0, 5) + encoded, err := exact.Encode("next") + require.NoError(t, err) + assert.Equal(t, "next\n", string(encoded)) + + region, bodyStart, err := splitFrontMatter([]byte("\ufeff---\r\ndescription: value\r\n---\r\n\r\nBody")) + require.NoError(t, err) + assert.NotEmpty(t, region.Data) + assert.Greater(t, bodyStart, 0) +} + +func TestFrontMatterFenceMustStartTheLine(t *testing.T) { + source := []byte("---\ndescription: |\n ---\n not a fence\nname: support\n---\nBody\n") + root, err := parse(adapter.Document{Path: "rule.md", Data: source}, false) + require.NoError(t, err) + assert.Equal(t, "---\nnot a fence\n", root.Fields["frontmatter"].Fields["description"].Value) + assert.Equal(t, "support", root.Fields["frontmatter"].Fields["name"].Value) + assert.Equal(t, "Body", root.Fields["body"].Value) +} + +func TestMarkersInsideFencedCodeDoNotCount(t *testing.T) { + source := []byte("# Guide\n\n```markdown\n\nshown as an example\n\n```\n\n\nReal region.\n\n") + root, err := parse(adapter.Document{Path: "AGENTS.md", Data: source}, false) + require.NoError(t, err) + markers := root.Fields["markers"].Items + require.Len(t, markers, 1) + assert.Equal(t, "real", markers[0].Fields["id"].Value) + assert.Equal(t, "Real region.", markers[0].Fields["body"].Value) +} + +func TestClosingFenceHasNothingAfterIt(t *testing.T) { + source := []byte("# Guide\n\n```\n# Code heading\n```js\n\nstill code\n\n``` \n\n## Real\n\n\nReal region.\n\n") + root, err := parse(adapter.Document{Path: "AGENTS.md", Data: source}, false) + require.NoError(t, err) + markers := root.Fields["markers"].Items + require.Len(t, markers, 1) + assert.Equal(t, "real", markers[0].Fields["id"].Value) + headings := make([]string, 0, 2) + for _, section := range root.Fields["sections"].Items { + headings = append(headings, section.Fields["heading"].Value.(string)) + } + assert.Equal(t, []string{"Guide", "Real"}, headings) + + source = []byte("``` a ` b\n# Heading\n") + root, err = parse(adapter.Document{Path: "AGENTS.md", Data: source}, false) + require.NoError(t, err) + require.Len(t, root.Fields["sections"].Items, 1) +} + +func TestMarkersInsideIndentedCodeDoNotCount(t *testing.T) { + source := []byte("# Guide\n\nExample:\n\n \n shown as an example\n \n\n\nReal region.\n\n") + root, err := parse(adapter.Document{Path: "AGENTS.md", Data: source}, false) + require.NoError(t, err) + markers := root.Fields["markers"].Items + require.Len(t, markers, 1) + assert.Equal(t, "real", markers[0].Fields["id"].Value) +} diff --git a/internal/sync/adapter/markdown/testdata/agent-skill/refunds/references/policy.md b/internal/sync/adapter/markdown/testdata/agent-skill/refunds/references/policy.md new file mode 100644 index 000000000..52b962517 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/agent-skill/refunds/references/policy.md @@ -0,0 +1,3 @@ +# Refund policy + +Escalate refunds above $500. diff --git a/internal/sync/adapter/markdown/testdata/agent-skill/refunds/scripts/check_refund.py b/internal/sync/adapter/markdown/testdata/agent-skill/refunds/scripts/check_refund.py new file mode 100644 index 000000000..e28fe6cd2 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/agent-skill/refunds/scripts/check_refund.py @@ -0,0 +1,2 @@ +def eligible(total: float) -> bool: + return total <= 500 diff --git a/internal/sync/adapter/markdown/testdata/agent-skill/refunds/skill-instructions.md.golden b/internal/sync/adapter/markdown/testdata/agent-skill/refunds/skill-instructions.md.golden new file mode 100644 index 000000000..95237b040 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/agent-skill/refunds/skill-instructions.md.golden @@ -0,0 +1,12 @@ +--- +name: refunds +description: Explain the refund policy and escalation rules. +license: Apache-2.0 +metadata: + owner: support-platform +--- + +# Refund handling + +Confirm the order ID first. +Escalate refunds above $500. diff --git a/internal/sync/adapter/markdown/testdata/agents-instructions.md.golden b/internal/sync/adapter/markdown/testdata/agents-instructions.md.golden new file mode 100644 index 000000000..16f2d0c47 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/agents-instructions.md.golden @@ -0,0 +1,9 @@ +# Repository guide + +## Support agent + +Ask for an order ID before discussing an order. + +## Build + +Run `make test`. diff --git a/internal/sync/adapter/markdown/testdata/profiles/agents-instructions.md.golden b/internal/sync/adapter/markdown/testdata/profiles/agents-instructions.md.golden new file mode 100644 index 000000000..28d9f4674 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/profiles/agents-instructions.md.golden @@ -0,0 +1,13 @@ +# Repository guide + +## Support + +Ask for an order ID. + + +Explain the refund policy. + + +## Build + +Run `make test`. diff --git a/internal/sync/adapter/markdown/testdata/profiles/claude-instructions.md.golden b/internal/sync/adapter/markdown/testdata/profiles/claude-instructions.md.golden new file mode 100644 index 000000000..f4a10263c --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/profiles/claude-instructions.md.golden @@ -0,0 +1,10 @@ +--- +name: support +description: Support subagent instructions +tools: Read, Grep +model: sonnet +--- + +# Support subagent + +Ask for an order ID and follow the support policy. diff --git a/internal/sync/adapter/markdown/testdata/profiles/copilot.instructions.md b/internal/sync/adapter/markdown/testdata/profiles/copilot.instructions.md new file mode 100644 index 000000000..024f8f369 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/profiles/copilot.instructions.md @@ -0,0 +1,7 @@ +--- +applyTo: "src/support/**/*.ts" +--- + +# Support + +Use the support policy. diff --git a/internal/sync/adapter/markdown/testdata/profiles/cursor-rule.mdc b/internal/sync/adapter/markdown/testdata/profiles/cursor-rule.mdc new file mode 100644 index 000000000..6eef5da26 --- /dev/null +++ b/internal/sync/adapter/markdown/testdata/profiles/cursor-rule.mdc @@ -0,0 +1,9 @@ +--- +description: Support instructions +globs: "src/support/**/*.ts" +alwaysApply: false +--- + +# Support + +Ask for an order ID before discussing an order. diff --git a/internal/sync/adapter/node.go b/internal/sync/adapter/node.go new file mode 100644 index 000000000..24cd82812 --- /dev/null +++ b/internal/sync/adapter/node.go @@ -0,0 +1,207 @@ +package adapter + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strconv" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// Node is one logical value that an adapter exposes to select and map paths. +// +// A leaf has a Value, the byte span that holds it, and an Encode function that +// writes a new value in the syntax of the source. An object has Fields. A +// collection has Items. A node whose Value is nil is unknown: the adapter saw +// it but cannot read it statically. A read-only node has a Reason, and an +// optional Code, that explain why sync cannot write it. +type Node struct { + Value any + Fields map[string]*Node + Items []*Node + Collection bool + Start int + End int + File string + Capability Capability + Reason string + Code syncdomain.ErrorCode + Encode func(any) ([]byte, error) +} + +// String returns a writable string leaf. encode writes a new string in the +// syntax of the source. +func String(value string, start, end int, encode func(string) ([]byte, error)) *Node { + return &Node{ + Value: value, Start: start, End: end, Capability: CapabilityDirect, + Encode: func(value any) ([]byte, error) { + text, ok := value.(string) + if !ok { + return nil, TypeMismatch("source", "a string") + } + return encode(text) + }, + } +} + +// Number returns a writable number leaf. +func Number(value float64, start, end int, encode func(float64) ([]byte, error)) *Node { + return &Node{ + Value: value, Start: start, End: end, Capability: CapabilityDirect, + Encode: func(value any) ([]byte, error) { + number, ok := AsNumber(value) + if !ok { + return nil, TypeMismatch("source", "a number") + } + return encode(number) + }, + } +} + +// Bool returns a writable boolean leaf. encode writes a new boolean in the +// syntax of the source. The default writes true or false. +func Bool(value bool, start, end int, encode func(bool) ([]byte, error)) *Node { + if encode == nil { + encode = func(value bool) ([]byte, error) { return []byte(strconv.FormatBool(value)), nil } + } + return &Node{ + Value: value, Start: start, End: end, Capability: CapabilityDirect, + Encode: func(value any) ([]byte, error) { + boolean, ok := value.(bool) + if !ok { + return nil, TypeMismatch("source", "a boolean") + } + return encode(boolean) + }, + } +} + +// ReadOnly returns a node that sync can see but cannot write. +func ReadOnly(start, end int, reason string) *Node { + return &Node{Start: start, End: end, Capability: CapabilityReadOnly, Reason: reason} +} + +// Collections returns the root of a code source: one named collection of +// resources for each logical kind, such as agents, prompts, tools, and +// constants. +func Collections(collections map[string][]*Node) *Node { + fields := make(map[string]*Node, len(collections)) + for name, items := range collections { + fields[name] = &Node{Items: items, Collection: true} + } + return &Node{Fields: fields} +} + +// Resource returns one logical resource with its id field set, so that a +// keyed selector such as agents[id="support"] can find it. +func Resource(id string, fields map[string]*Node) *Node { + fields["id"] = &Node{Value: id} + return &Node{Fields: fields} +} + +// IsCollection reports whether a path can index the node. +func (node *Node) IsCollection() bool { + if node == nil { + return false + } + if node.Collection || node.Items != nil { + return true + } + value := reflect.ValueOf(node.Value) + return value.IsValid() && (value.Kind() == reflect.Array || value.Kind() == reflect.Slice) +} + +// MarkReadOnly makes the node and every node below it read-only. +func (node *Node) MarkReadOnly(reason string) { + if node == nil { + return + } + node.Capability, node.Reason, node.Encode = CapabilityReadOnly, reason, nil + for _, child := range node.Fields { + child.MarkReadOnly(reason) + } + for _, child := range node.Items { + child.MarkReadOnly(reason) + } +} + +// SetFile records the file that holds the node and every node below it that +// has no file yet. An imported value keeps the file that it came from. +func (node *Node) SetFile(file string) { + node.setFile(file, map[*Node]struct{}{}) +} + +func (node *Node) setFile(file string, visited map[*Node]struct{}) { + if node == nil { + return + } + if _, seen := visited[node]; seen { + return + } + visited[node] = struct{}{} + if node.File == "" { + node.File = file + } + for _, child := range node.Fields { + child.setFile(file, visited) + } + for _, child := range node.Items { + child.setFile(file, visited) + } +} + +// AsNumber returns the float64 value of a number from any parser. +func AsNumber(value any) (float64, bool) { + switch value := value.(type) { + case float64: + return value, true + case int: + return float64(value), true + case int64: + return float64(value), true + default: + return 0, false + } +} + +// TypeMismatch is the problem of a mapped field whose value has the wrong type. +func TypeMismatch(field, expected string) error { + return syncdomain.NewProblem( + syncdomain.ErrorMapTypeMismatch, + fmt.Sprintf("mapped field %q needs %s", field, expected), + "Map the field to a compatible static value.", + ) +} + +// JSONEncoder encodes a value as JSON in the layout of the raw text that it +// replaces. A value that spanned lines keeps its indentation unit and the +// indentation of its closing line. A value on one line stays on one line. +func JSONEncoder(raw []byte) func(any) ([]byte, error) { + return func(next any) ([]byte, error) { + unit, prefix, multiline := jsonIndentation(raw) + if !multiline { + return json.Marshal(next) + } + return json.MarshalIndent(next, prefix, unit) + } +} + +// jsonIndentation returns the indentation unit and the indentation of the +// closing line of a multi-line JSON value. +func jsonIndentation(raw []byte) (unit, prefix string, multiline bool) { + lines := bytes.Split(raw, []byte("\n")) + if len(lines) < 2 { + return "", "", false + } + last := lines[len(lines)-1] + prefix = string(last[:len(last)-len(bytes.TrimLeft(last, " \t"))]) + for _, line := range lines[1 : len(lines)-1] { + indent := line[:len(line)-len(bytes.TrimLeft(line, " \t"))] + if len(indent) > len(prefix) { + return string(indent[len(prefix):]), prefix, true + } + } + return " ", prefix, true +} diff --git a/internal/sync/adapter/parse.go b/internal/sync/adapter/parse.go new file mode 100644 index 000000000..c38f3799d --- /dev/null +++ b/internal/sync/adapter/parse.go @@ -0,0 +1,324 @@ +package adapter + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// ParseSource reads one mapped resource from current source bytes. A +// variation projection has the mode that the source implies. The planner +// converts it to the mode of the config. +func ParseSource( + bundle SourceBundle, + id syncdomain.ResourceID, + reference syncdomain.Reference, +) (syncdomain.Projection, error) { + projection, _, err := ParseSourceWithSelections(bundle, id, reference) + return projection, err +} + +// ParseSourceWithSelections reads one mapped resource and the identity of +// each indexed item on its paths. +func ParseSourceWithSelections( + bundle SourceBundle, + id syncdomain.ResourceID, + reference syncdomain.Reference, +) (syncdomain.Projection, []syncdomain.IndexedSelection, error) { + reference, err := NormalizeReference(id, reference) + if err != nil { + return nil, nil, err + } + root, err := Parse(bundle, reference.Format) + if err != nil { + return nil, nil, annotateProblem(err, id, bundle.Primary.Path) + } + selected, selectedIndexes, err := root.resolve(reference.Select, "select") + if err != nil { + return nil, nil, annotateProblem(fmt.Errorf("select %q: %w", reference.Select, err), id, bundle.Primary.Path) + } + selections, err := indexedSelections(selectedIndexes, selected) + if err != nil { + return nil, nil, annotateProblem(err, id, bundle.Primary.Path) + } + fields := make([]string, 0, len(reference.Map)) + for field := range reference.Map { + fields = append(fields, field) + } + slices.Sort(fields) + for _, field := range fields { + _, indexes, err := selected.resolve(reference.Map[field], "map."+field) + if err != nil { + return nil, nil, annotateProblem(fmt.Errorf("map field %q: %w", field, err), id, bundle.Primary.Path) + } + mappedSelections, err := indexedSelections(indexes, selected) + if err != nil { + return nil, nil, annotateProblem(err, id, bundle.Primary.Path) + } + selections = append(selections, mappedSelections...) + } + slices.SortFunc(selections, func(left, right syncdomain.IndexedSelection) int { + return strings.Compare(left.Path, right.Path) + }) + projection, err := buildProjection(id.Kind, selected, reference.Map) + if variation, ok := projection.(syncdomain.VariationProjection); ok { + if reference.Tools != nil { + keys := attachmentKeys(*reference.Tools) + variation.Tools = &keys + variation.FieldStates[syncdomain.FieldTools] = syncdomain.FieldState{ + Capability: syncdomain.CapabilityDirect, Known: true, + } + } + if reference.Skills != nil { + keys := attachmentKeys(*reference.Skills) + variation.Skills = &keys + variation.FieldStates[syncdomain.FieldSkills] = syncdomain.FieldState{ + Capability: syncdomain.CapabilityDirect, Known: true, + } + } + projection = variation + } + return projection, selections, annotateProblem(err, id, bundle.Primary.Path) +} + +// buildProjection reads each mapped field below the selected node into the +// typed projection of the kind. +func buildProjection( + kind syncdomain.Kind, + selected *Node, + fieldMap map[string]string, +) (syncdomain.Projection, error) { + switch kind { + case syncdomain.KindVariation: + _, promptMapped := fieldMap[string(syncdomain.FieldPrompt)] + _, messagesMapped := fieldMap[string(syncdomain.FieldMessages)] + switch { + case !promptMapped && !messagesMapped: + return nil, fmt.Errorf("map field %q is required for a variation", syncdomain.FieldPrompt) + case promptMapped && messagesMapped: + // Both names fill the one prompt of the variation, so a second one + // would overwrite or shadow the first. + return nil, fmt.Errorf( + "map fields %q and %q cannot both be set for a variation", + syncdomain.FieldPrompt, syncdomain.FieldMessages, + ) + } + projection := syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, FieldStates: make(map[syncdomain.Field]syncdomain.FieldState), + } + for name, path := range fieldMap { + node, err := mappedNode(selected, name, path) + if err != nil { + return nil, err + } + field := syncdomain.Field(name) + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.Capability, Known: node.Value != nil, Reason: node.Reason, Code: node.Code, File: node.File, + } + parameterKey, parameterField := syncdomain.ModelParameterKey(field) + if parameterField { + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.Capability, Known: true, Reason: node.Reason, Code: node.Code, File: node.File, + } + value, ok := jsonValue(node.Value) + if !ok { + return nil, TypeMismatch(name, "a JSON-compatible value") + } + if projection.Parameters == nil { + parameters := map[string]any{} + projection.Parameters = ¶meters + } + (*projection.Parameters)[parameterKey] = value + continue + } + if field == syncdomain.FieldModelParameters && node.Value == nil { + return nil, TypeMismatch(name, "an object") + } + if node.Value == nil { + continue + } + switch field { + case syncdomain.FieldPrompt, syncdomain.FieldMessages: + switch value := syncdomain.NormalizePrompt(node.Value).(type) { + case string: + projection.Instructions = &value + case []syncdomain.Message: + for _, message := range value { + if !syncdomain.ValidMessageRole(message.Role) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorMapTypeMismatch, + fmt.Sprintf("message role %q is not supported", message.Role), + "Use system, user, or assistant.", + ) + } + } + projection.Mode = syncdomain.VariationModeCompletion + projection.Messages = &value + default: + return nil, TypeMismatch(name, "a string or a message list") + } + case syncdomain.FieldName: + value, ok := node.Value.(string) + if !ok { + return nil, TypeMismatch(name, "a string") + } + projection.Name = &value + case syncdomain.FieldModel: + value, ok := node.Value.(string) + if !ok { + return nil, TypeMismatch(name, "a string") + } + projection.Model = &value + case syncdomain.FieldModelParameters: + value, ok := objectValue(node.Value) + if !ok { + return nil, TypeMismatch(name, "an object") + } + value = cloneObject(value) + delete(value, "tools") + projection.Parameters = &value + } + } + return projection, nil + case syncdomain.KindTool: + projection := syncdomain.ToolProjection{FieldStates: make(map[syncdomain.Field]syncdomain.FieldState)} + for name, path := range fieldMap { + node, err := mappedNode(selected, name, path) + if err != nil { + return nil, err + } + field := syncdomain.Field(name) + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.Capability, Known: node.Value != nil, Reason: node.Reason, Code: node.Code, File: node.File, + } + if node.Value == nil { + continue + } + switch field { + case syncdomain.FieldDescription: + value, ok := node.Value.(string) + if !ok { + return nil, TypeMismatch(name, "a string") + } + projection.Description = &value + case syncdomain.FieldSchema: + value, ok := objectValue(node.Value) + if !ok { + return nil, TypeMismatch(name, "an object") + } + projection.Schema = &value + case syncdomain.FieldCustomParameters: + value, ok := objectValue(node.Value) + if !ok { + return nil, TypeMismatch(name, "an object") + } + projection.CustomParameters = &value + case syncdomain.FieldTags: + value, ok := stringList(node.Value) + if !ok { + return nil, TypeMismatch(name, "a string collection") + } + projection.Tags = &value + } + } + return projection, nil + case syncdomain.KindSkill: + projection := syncdomain.SkillProjection{FieldStates: make(map[syncdomain.Field]syncdomain.FieldState)} + for name, path := range fieldMap { + node, err := mappedNode(selected, name, path) + if err != nil { + return nil, err + } + field := syncdomain.Field(name) + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.Capability, Known: node.Value != nil, Reason: node.Reason, Code: node.Code, File: node.File, + } + if node.Value == nil { + continue + } + value, ok := node.Value.(string) + if !ok { + return nil, TypeMismatch(name, "a string") + } + if field == syncdomain.FieldDescription { + projection.Description = &value + } else { + projection.Markdown = &value + } + } + return projection, nil + default: + return nil, fmt.Errorf("unsupported resource kind %q", kind) + } +} + +func mappedNode(root *Node, name, path string) (*Node, error) { + node, err := root.Resolve(path) + if err != nil { + return nil, fmt.Errorf("map field %q: %w", name, err) + } + return node, nil +} + +func attachmentKeys(refs []syncdomain.AttachmentRef) []string { + keys := make([]string, len(refs)) + for index := range refs { + keys[index] = refs[index].Key + } + slices.Sort(keys) + return keys +} + +func jsonValue(value any) (any, bool) { + encoded, err := json.Marshal(value) + if err != nil { + return nil, false + } + var decoded any + if err := json.Unmarshal(encoded, &decoded); err != nil { + return nil, false + } + return decoded, true +} + +func objectValue(value any) (map[string]any, bool) { + object, ok := value.(map[string]any) + if !ok { + return nil, false + } + return object, true +} + +func cloneObject(value map[string]any) map[string]any { + cloned, ok := jsonValue(value) + if !ok { + return map[string]any{} + } + object, ok := cloned.(map[string]any) + if !ok { + return map[string]any{} + } + return object +} + +func stringList(value any) ([]string, bool) { + switch values := value.(type) { + case []string: + return slices.Clone(values), true + case []any: + result := make([]string, len(values)) + for index, value := range values { + text, ok := value.(string) + if !ok { + return nil, false + } + result[index] = text + } + return result, true + default: + return nil, false + } +} diff --git a/internal/sync/adapter/path.go b/internal/sync/adapter/path.go new file mode 100644 index 000000000..767aef38c --- /dev/null +++ b/internal/sync/adapter/path.go @@ -0,0 +1,270 @@ +package adapter + +import ( + "encoding/json" + "fmt" + "strconv" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type sourcePath struct { + root bool + steps []pathStep +} + +type pathStep struct { + key string + filters []pathFilter + index *int +} + +type pathFilter struct { + key string + value string +} + +// ParsePath checks the shared select and map path grammar. +func ParsePath(text string) error { + _, err := parsePath(text) + return err +} + +func parsePath(text string) (sourcePath, error) { + if text == "$" { + return sourcePath{root: true}, nil + } + if text == "" { + return sourcePath{}, pathSyntaxError(text, 0, "path is empty") + } + + parser := pathParser{text: text} + key, err := parser.readKey() + if err != nil { + return sourcePath{}, err + } + result := sourcePath{steps: []pathStep{{key: key}}} + for parser.offset < len(text) { + switch text[parser.offset] { + case '.': + parser.offset++ + key, err = parser.readIdentifier() + if err != nil { + return sourcePath{}, err + } + result.steps = append(result.steps, pathStep{key: key}) + case '[': + step, kind, err := parser.readBracket() + if err != nil { + return sourcePath{}, err + } + if kind == bracketFilter { + last := len(result.steps) - 1 + if result.steps[last].index != nil { + return sourcePath{}, pathSyntaxError(text, parser.offset, "a selector must follow a field") + } + if len(result.steps[last].filters) != 0 { + return sourcePath{}, pathSyntaxError(text, parser.offset, "a path step can have only one selector") + } + result.steps[last].filters = step.filters + } else { + result.steps = append(result.steps, step) + } + default: + return sourcePath{}, pathSyntaxError(text, parser.offset, "expected a dot or bracket") + } + } + return result, nil +} + +type pathParser struct { + text string + offset int +} + +type bracketKind int + +const ( + bracketField bracketKind = iota + bracketFilter + bracketIndex +) + +func (parser *pathParser) readKey() (string, error) { + if parser.offset < len(parser.text) && parser.text[parser.offset] == '[' { + step, kind, err := parser.readBracket() + if err != nil { + return "", err + } + if kind != bracketField { + return "", pathSyntaxError(parser.text, parser.offset, "a path cannot start with a selector or index") + } + return step.key, nil + } + return parser.readIdentifier() +} + +func (parser *pathParser) readIdentifier() (string, error) { + start := parser.offset + if start >= len(parser.text) || !identifierStart(parser.text[start]) { + return "", pathSyntaxError(parser.text, start, "expected an identifier") + } + parser.offset++ + for parser.offset < len(parser.text) && identifierContinue(parser.text[parser.offset]) { + parser.offset++ + } + return parser.text[start:parser.offset], nil +} + +func (parser *pathParser) readBracket() (pathStep, bracketKind, error) { + start := parser.offset + parser.offset++ + if parser.offset < len(parser.text) && parser.text[parser.offset] >= '0' && parser.text[parser.offset] <= '9' { + return parser.readIndex(start) + } + key, err := parser.readBracketKey() + if err != nil { + return pathStep{}, bracketField, err + } + if parser.offset >= len(parser.text) { + return pathStep{}, bracketField, pathSyntaxError(parser.text, start, "unclosed bracket") + } + if parser.text[parser.offset] == ']' { + parser.offset++ + return pathStep{key: key}, bracketField, nil + } + if parser.text[parser.offset] != '=' { + return pathStep{}, bracketField, pathSyntaxError(parser.text, parser.offset, "expected equals or closing bracket") + } + parser.offset++ + if parser.offset >= len(parser.text) || parser.text[parser.offset] != '"' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "selector value must be a JSON string") + } + value, err := parser.readJSONString() + if err != nil { + return pathStep{}, bracketFilter, err + } + filters := []pathFilter{{key: key, value: value}} + for parser.offset < len(parser.text) && parser.text[parser.offset] == ',' { + parser.offset++ + nextKey, err := parser.readBracketKey() + if err != nil { + return pathStep{}, bracketFilter, err + } + if parser.offset >= len(parser.text) || parser.text[parser.offset] != '=' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "expected equals") + } + parser.offset++ + if parser.offset >= len(parser.text) || parser.text[parser.offset] != '"' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "selector value must be a JSON string") + } + nextValue, err := parser.readJSONString() + if err != nil { + return pathStep{}, bracketFilter, err + } + filters = append(filters, pathFilter{key: nextKey, value: nextValue}) + } + if parser.offset >= len(parser.text) || parser.text[parser.offset] != ']' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "expected a closing bracket") + } + parser.offset++ + return pathStep{filters: filters}, bracketFilter, nil +} + +func (parser *pathParser) readIndex(start int) (pathStep, bracketKind, error) { + numberStart := parser.offset + for parser.offset < len(parser.text) && parser.text[parser.offset] >= '0' && parser.text[parser.offset] <= '9' { + parser.offset++ + } + raw := parser.text[numberStart:parser.offset] + if len(raw) > 1 && raw[0] == '0' { + return pathStep{}, bracketIndex, pathSyntaxError(parser.text, numberStart, "an index cannot have a leading zero") + } + if parser.offset >= len(parser.text) || parser.text[parser.offset] != ']' { + return pathStep{}, bracketIndex, pathSyntaxError(parser.text, parser.offset, "an index must be a decimal integer") + } + value, err := strconv.ParseUint(raw, 10, 31) + if err != nil { + return pathStep{}, bracketIndex, pathSyntaxError(parser.text, numberStart, "index exceeds 2147483647") + } + parser.offset++ + index := int(value) + return pathStep{index: &index}, bracketIndex, nil +} + +func (parser *pathParser) readBracketKey() (string, error) { + if parser.offset < len(parser.text) && parser.text[parser.offset] == '"' { + return parser.readJSONString() + } + return parser.readIdentifier() +} + +func (parser *pathParser) readJSONString() (string, error) { + start := parser.offset + parser.offset++ + for parser.offset < len(parser.text) { + switch parser.text[parser.offset] { + case '\\': + parser.offset += 2 + case '"': + parser.offset++ + quoted := parser.text[start:parser.offset] + if err := rejectUnpairedSurrogate(quoted); err != nil { + return "", pathSyntaxError(parser.text, start, err.Error()) + } + var value string + if err := json.Unmarshal([]byte(quoted), &value); err != nil { + return "", pathSyntaxError(parser.text, start, "invalid JSON string") + } + return value, nil + default: + parser.offset++ + } + } + return "", pathSyntaxError(parser.text, start, "unclosed JSON string") +} + +func rejectUnpairedSurrogate(quoted string) error { + for index := 0; index+5 < len(quoted); index++ { + if quoted[index:index+2] != `\u` { + continue + } + code, err := strconv.ParseUint(quoted[index+2:index+6], 16, 16) + if err != nil { + continue + } + switch { + case code >= 0xd800 && code <= 0xdbff: + if index+11 >= len(quoted) || quoted[index+6:index+8] != `\u` { + return fmt.Errorf("unpaired Unicode surrogate") + } + low, err := strconv.ParseUint(quoted[index+8:index+12], 16, 16) + if err != nil || low < 0xdc00 || low > 0xdfff { + return fmt.Errorf("unpaired Unicode surrogate") + } + index += 6 + case code >= 0xdc00 && code <= 0xdfff: + return fmt.Errorf("unpaired Unicode surrogate") + } + } + return nil +} + +func pathSyntaxError(path string, offset int, reason string) error { + if offset > len(path) { + offset = len(path) + } + return syncdomain.NewProblem( + syncdomain.ErrorInvalidPathSyntax, + fmt.Sprintf("invalid path at byte %d: %s", offset, reason), + "Use $, named fields, zero-based indexes, keyed selectors, or quoted literal keys.", + ) +} + +func identifierStart(value byte) bool { + return value == '_' || value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z' +} + +func identifierContinue(value byte) bool { + return identifierStart(value) || value >= '0' && value <= '9' +} diff --git a/internal/sync/adapter/reference.go b/internal/sync/adapter/reference.go new file mode 100644 index 000000000..b20446631 --- /dev/null +++ b/internal/sync/adapter/reference.go @@ -0,0 +1,141 @@ +package adapter + +import ( + "fmt" + "slices" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// NormalizeReference validates a descriptor and fills conventional defaults. +func NormalizeReference(id syncdomain.ResourceID, reference syncdomain.Reference) (syncdomain.Reference, error) { + if reference.File == "" { + return syncdomain.Reference{}, invalidReference(id, "file is required", "Pass --file.") + } + registered, err := lookup(reference.Format) + if err != nil { + return syncdomain.Reference{}, err + } + defaultSelect, defaultMap := registered.Defaults(id.Kind) + if reference.Select == "" { + if defaultSelect == "" { + return syncdomain.Reference{}, invalidReference( + id, fmt.Sprintf("select is required for format %q", reference.Format), "Pass --select.", + ) + } + reference.Select = defaultSelect + } + if err := ParsePath(reference.Select); err != nil { + return syncdomain.Reference{}, annotateProblem( + fmt.Errorf("select %q: %w", reference.Select, err), id, reference.File, + ) + } + if len(reference.Map) == 0 { + if len(defaultMap) == 0 { + return syncdomain.Reference{}, invalidReference( + id, "map is required", "Pass at least one --map field=path value.", + ) + } + reference.Map = defaultMap + } + for target := range reference.Map { + if err := ParsePath(target); err != nil { + return syncdomain.Reference{}, annotateProblem( + fmt.Errorf("map target %q: %w", target, err), id, reference.File, + ) + } + } + fields, err := syncdomain.ProjectionFields(reference) + if err != nil { + return syncdomain.Reference{}, invalidReference(id, err.Error(), "Use a field valid for this resource kind.") + } + wholeParameters := false + parameterKeys := map[string]syncdomain.Field{} + for _, field := range fields { + if field == syncdomain.FieldTools || field == syncdomain.FieldSkills { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("map key %q is not allowed", field), + fmt.Sprintf("List %s keys under top-level %q.", field, field), + ) + } + if !fieldAllowed(id.Kind, field) { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("field %q is not valid for %s", field, id.Kind), + "Use a mapped field supported by this resource kind.", + ) + } + if field == syncdomain.FieldModelParameters { + wholeParameters = true + } + if key, parameter := syncdomain.ModelParameterKey(field); parameter { + if key == "tools" { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("map target %q duplicates the tools field", field), + "List tool keys under top-level \"tools\".", + ) + } + if previous, duplicate := parameterKeys[key]; duplicate { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("map targets %q and %q select the same model parameter", previous, field), + "Map each model parameter key once.", + ) + } + parameterKeys[key] = field + } + if err := ParsePath(reference.Map[string(field)]); err != nil { + return syncdomain.Reference{}, annotateProblem( + fmt.Errorf("map field %q: %w", field, err), id, reference.File, + ) + } + } + if wholeParameters && len(parameterKeys) != 0 { + return syncdomain.Reference{}, invalidReference( + id, + "map target \"model.parameters\" overlaps an individual model parameter target", + "Map the full parameters object or individual parameter keys.", + ) + } + return reference, nil +} + +func invalidReference(id syncdomain.ResourceID, message, suggestion string) error { + return syncdomain.NewProblem( + syncdomain.ErrorInvalidReference, message, suggestion, + ).WithResource(id) +} + +func fieldAllowed(kind syncdomain.Kind, field syncdomain.Field) bool { + switch kind { + case syncdomain.KindVariation: + if _, parameter := syncdomain.ModelParameterKey(field); parameter { + return true + } + return slices.Contains([]syncdomain.Field{ + syncdomain.FieldPrompt, syncdomain.FieldMessages, syncdomain.FieldName, syncdomain.FieldModel, + syncdomain.FieldModelParameters, + }, field) + case syncdomain.KindTool: + return slices.Contains([]syncdomain.Field{ + syncdomain.FieldDescription, syncdomain.FieldSchema, + syncdomain.FieldCustomParameters, syncdomain.FieldTags, + }, field) + case syncdomain.KindSkill: + return field == syncdomain.FieldDescription || field == syncdomain.FieldMarkdown + default: + return false + } +} + +func annotateProblem(err error, id syncdomain.ResourceID, file string) error { + if err == nil { + return nil + } + if problem, ok := syncdomain.ProblemFrom(err); ok { + _ = problem.WithResource(id).WithFile(file, 0, 0) + } + return err +} diff --git a/internal/sync/adapter/resolve.go b/internal/sync/adapter/resolve.go new file mode 100644 index 000000000..0b769cf82 --- /dev/null +++ b/internal/sync/adapter/resolve.go @@ -0,0 +1,232 @@ +package adapter + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "slices" + "strings" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// Resolve returns the node at a select or map path below root. +func (root *Node) Resolve(text string) (*Node, error) { + node, _, err := root.resolve(text, "") + return node, err +} + +// indexedResolution is one index step on a resolved path: where it is, how +// long the collection was, and which item it chose. +type indexedResolution struct { + path string + collectionLength int + item *Node +} + +// resolve returns the node at a path and the index steps that it took. scope +// names the path in the identity of each index step. +func (root *Node) resolve(text, scope string) (*Node, []indexedResolution, error) { + path, err := parsePath(text) + if err != nil { + return nil, nil, err + } + if path.root { + return root, nil, nil + } + current := root + var indexes []indexedResolution + for _, step := range path.steps { + if step.index != nil { + if !current.IsCollection() { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathTypeMismatch, + fmt.Sprintf("index %d needs a collection", *step.index), + "Select a collection before this index.", + ) + } + if *step.index >= len(current.Items) { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorIndexOutOfRange, + fmt.Sprintf("index %d is out of range for collection of length %d", *step.index, len(current.Items)), + "Use an index within the collection.", + ) + } + collectionLength := len(current.Items) + current = current.Items[*step.index] + indexes = append(indexes, indexedResolution{ + path: fmt.Sprintf("%s#%d", scope, len(indexes)), + collectionLength: collectionLength, + item: current, + }) + continue + } + if current.Fields == nil { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathTypeMismatch, + fmt.Sprintf("%q does not contain fields", step.key), + "Select an object before this field.", + ) + } + next, ok := current.Fields[step.key] + if !ok { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathNotFound, + fmt.Sprintf("field %q was not found", step.key), + "Update the select or map path.", + ) + } + current = next + if len(step.filters) == 0 { + continue + } + var matches []*Node + for _, item := range current.Items { + matched := true + for _, filter := range step.filters { + identity, ok := item.Fields[filter.key] + if !ok { + matched = false + break + } + text, ok := identity.Value.(string) + if !ok { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorFilterTypeMismatch, + fmt.Sprintf("selector field %q is not a string", filter.key), + "Select a direct string identity field.", + ) + } + if text != filter.value { + matched = false + break + } + } + if matched { + matches = append(matches, item) + } + } + selector := formatFilters(step.filters) + switch len(matches) { + case 0: + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathNotFound, + fmt.Sprintf("selector %s matched no values", selector), + "Update the selector or restore the source value.", + ) + case 1: + current = matches[0] + default: + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathAmbiguous, + fmt.Sprintf("selector %s matched %d values", selector, len(matches)), + "Use a selector that matches one resource.", + ) + } + } + return current, indexes, nil +} + +// indexedSelections records the identity of each indexed item, so that a +// later sync can tell when the item at that position changed. +func indexedSelections(indexes []indexedResolution, selectedRoot *Node) ([]syncdomain.IndexedSelection, error) { + selections := make([]syncdomain.IndexedSelection, 0, len(indexes)) + var ancestors []string + for _, index := range indexes { + identity := directSelectionIdentity(index.item) + if identity == nil { + // Without scalar identity, bind the nearest selected resource root and prior indexed ancestors. + identity = struct { + Root any `json:"root"` + Ancestors []string `json:"ancestors,omitempty"` + Item any `json:"item"` + }{ + Root: stableNodeValue(selectedRoot, map[*Node]bool{}), + Ancestors: slices.Clone(ancestors), + Item: stableNodeValue(index.item, map[*Node]bool{}), + } + } + fingerprint, err := selectionFingerprint(identity) + if err != nil { + return nil, err + } + selections = append(selections, syncdomain.IndexedSelection{ + Path: index.path, CollectionLength: index.collectionLength, Fingerprint: fingerprint, + }) + ancestors = append(ancestors, fingerprint) + } + return selections, nil +} + +func directSelectionIdentity(node *Node) any { + if node == nil || node.Fields == nil { + return nil + } + identity := map[string]any{} + for _, key := range []string{"id", "name", "role", "type"} { + field, ok := node.Fields[key] + if !ok || !scalarValue(field.Value) { + continue + } + identity[key] = field.Value + } + if len(identity) == 0 { + return nil + } + return identity +} + +func scalarValue(value any) bool { + switch value.(type) { + case string, bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: + return true + default: + return false + } +} + +func stableNodeValue(node *Node, visited map[*Node]bool) any { + if node == nil { + return nil + } + if node.Value != nil { + return node.Value + } + if visited[node] { + return "" + } + visited[node] = true + defer delete(visited, node) + if node.Fields != nil { + value := make(map[string]any, len(node.Fields)) + for key, field := range node.Fields { + value[key] = stableNodeValue(field, visited) + } + return value + } + if node.IsCollection() { + value := make([]any, len(node.Items)) + for index, item := range node.Items { + value[index] = stableNodeValue(item, visited) + } + return value + } + return nil +} + +func selectionFingerprint(value any) (string, error) { + canonical, err := json.Marshal(value) + if err != nil { + return "", fmt.Errorf("encode indexed selection fingerprint: %w", err) + } + sum := sha256.Sum256(canonical) + return fmt.Sprintf("sha256:%x", sum), nil +} + +func formatFilters(filters []pathFilter) string { + parts := make([]string, len(filters)) + for index, filter := range filters { + parts[index] = fmt.Sprintf("%s=%q", filter.key, filter.value) + } + return strings.Join(parts, ",") +} diff --git a/internal/sync/adapter/update.go b/internal/sync/adapter/update.go new file mode 100644 index 000000000..f9b64793d --- /dev/null +++ b/internal/sync/adapter/update.go @@ -0,0 +1,167 @@ +package adapter + +import ( + "bytes" + "encoding/json" + "fmt" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// UpdateSource plans the exact edits that give the mapped fields of a source +// the values of the desired projection. A field whose value is the same has +// no entry. A field that the source cannot write has a read-only entry. +func UpdateSource( + bundle SourceBundle, + id syncdomain.ResourceID, + reference syncdomain.Reference, + desired syncdomain.Projection, +) (UpdateResult, error) { + reference, err := NormalizeReference(id, reference) + if err != nil { + return UpdateResult{}, err + } + if desired.Kind() != id.Kind { + return UpdateResult{}, fmt.Errorf("projection kind %q does not match %s", desired.Kind(), id.Label()) + } + root, err := Parse(bundle, reference.Format) + if err != nil { + return UpdateResult{}, annotateProblem(err, id, bundle.Primary.Path) + } + selected, err := root.Resolve(reference.Select) + if err != nil { + return UpdateResult{}, annotateProblem(fmt.Errorf("select %q: %w", reference.Select, err), id, bundle.Primary.Path) + } + + result := UpdateResult{Fields: make(map[syncdomain.Field]FieldUpdate)} + for name, sourcePath := range reference.Map { + field := syncdomain.Field(name) + wanted, present := desired.Value(field) + if !present { + continue + } + current, err := selected.Resolve(sourcePath) + if err != nil { + return UpdateResult{}, fmt.Errorf("%s: map field %q: %w", bundle.Primary.Path, field, err) + } + if field == syncdomain.FieldPrompt || field == syncdomain.FieldMessages { + if semanticEqual(syncdomain.NormalizePrompt(current.Value), syncdomain.NormalizePrompt(wanted)) { + continue + } + if update, handled := updatePrompt(bundle, current, wanted); handled { + if len(update.Edits) != 0 || update.Capability != CapabilityDirect { + result.Fields[field] = update + } + continue + } + } + if semanticEqual(current.Value, wanted) { + continue + } + result.Fields[field] = updateLeaf(bundle, current, wanted) + } + return result, nil +} + +// updatePrompt reconciles the two prompt shapes. A text source holds one +// system message. A message list source changes the content of each message +// in place. handled is false when the leaf update applies. +func updatePrompt(bundle SourceBundle, current *Node, wanted any) (update FieldUpdate, handled bool) { + have, sourceIsMessages := current.Value.([]syncdomain.Message) + want, wantIsMessages := wanted.([]syncdomain.Message) + switch { + case sourceIsMessages: + if text, ok := wanted.(string); ok { + want = []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: text}} + } + return updateMessageContents(bundle, current, have, want), true + case wantIsMessages: + if len(want) != 1 || want[0].Role != syncdomain.RoleSystem { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: "text sources support one system message"}, true + } + return updateLeaf(bundle, current, want[0].Content), true + default: + return FieldUpdate{}, false + } +} + +// updateLeaf plans the edit of one leaf value. +func updateLeaf(bundle SourceBundle, current *Node, wanted any) FieldUpdate { + if current.Capability != CapabilityDirect || current.Encode == nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: current.Reason} + } + replacement, err := current.Encode(wanted) + if err != nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: err.Error()} + } + document, ok := bundleDocument(bundle, current.File) + if !ok { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: "source dependency is not available"} + } + return FieldUpdate{Capability: CapabilityDirect, Edits: []TextEdit{{ + File: document.Path, + Start: current.Start, + End: current.End, + Expected: bytes.Clone(document.Data[current.Start:current.End]), + Replacement: replacement, + }}} +} + +func updateMessageContents( + bundle SourceBundle, + current *Node, + have, want []syncdomain.Message, +) FieldUpdate { + if len(have) != len(want) { + return FieldUpdate{ + Capability: CapabilityReadOnly, + Reason: "add or remove messages by hand", + } + } + var edits []TextEdit + for index := range have { + if have[index].Role != want[index].Role { + return FieldUpdate{ + Capability: CapabilityReadOnly, + Reason: "reorder messages or change roles by hand", + } + } + if syncdomain.NormalizePromptText(have[index].Content) == syncdomain.NormalizePromptText(want[index].Content) { + continue + } + if index >= len(current.Items) || current.Items[index].Fields["content"] == nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: "message content is not a writable literal"} + } + content := current.Items[index].Fields["content"] + if content.Capability != CapabilityDirect || content.Encode == nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: content.Reason} + } + replacement, err := content.Encode(want[index].Content) + if err != nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: err.Error()} + } + document, ok := bundleDocument(bundle, content.File) + if !ok { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: "source dependency is not available"} + } + edits = append(edits, TextEdit{ + File: document.Path, Start: content.Start, End: content.End, + Expected: bytes.Clone(document.Data[content.Start:content.End]), Replacement: replacement, + }) + } + return FieldUpdate{Capability: CapabilityDirect, Edits: edits} +} + +func bundleDocument(bundle SourceBundle, file string) (Document, bool) { + if file == "" || file == bundle.Primary.Path { + return bundle.Primary, true + } + document, ok := bundle.Dependencies[file] + return document, ok +} + +func semanticEqual(left, right any) bool { + leftJSON, leftErr := json.Marshal(left) + rightJSON, rightErr := json.Marshal(right) + return leftErr == nil && rightErr == nil && bytes.Equal(leftJSON, rightJSON) +} diff --git a/internal/sync/adapter/yamltree/yamltree.go b/internal/sync/adapter/yamltree/yamltree.go new file mode 100644 index 000000000..c4df24ffd --- /dev/null +++ b/internal/sync/adapter/yamltree/yamltree.go @@ -0,0 +1,331 @@ +// Package yamltree builds logical nodes from a YAML document, with the exact +// byte span of each scalar. The YAML adapter and Markdown front matter use it. +package yamltree + +import ( + "bytes" + "encoding/json" + "fmt" + "math" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "gopkg.in/yaml.v3" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +// Region is a YAML document, or part of one, and the byte offset of its first +// byte in the file that contains it. +type Region struct { + Data []byte + StartOffset int +} + +// ScalarSpan returns the byte span and the raw text of one scalar. flow +// is true inside a flow sequence or a flow mapping, where a comma or a closing +// bracket ends a plain scalar. A plain scalar that spans lines has no exact +// span, so the caller treats it as read-only. +func ScalarSpan(region Region, node *yaml.Node, flow bool) (start, end int, raw string, exact bool, err error) { + lines := bytes.SplitAfter(region.Data, []byte("\n")) + if node.Line < 1 || node.Line > len(lines) || node.Column < 1 { + return 0, 0, "", false, fmt.Errorf("cannot locate YAML scalar") + } + start = region.StartOffset + for index := 0; index < node.Line-1; index++ { + start += len(lines[index]) + } + line := bytes.TrimRight(lines[node.Line-1], "\r\n") + localStart := byteOffsetForColumn(line, node.Column) + if localStart > len(line) { + return 0, 0, "", false, fmt.Errorf("cannot locate YAML scalar") + } + rest := line[localStart:] + switch { + case len(rest) != 0 && (rest[0] == '\'' || rest[0] == '"'): + var closed bool + end, closed := yamlQuotedEnd(rest) + raw, exact = string(rest[:end]), closed + default: + raw = string(yamlPlainScalar(rest, flow)) + exact = raw == node.Value + } + start += localStart + return start, start + len(raw), raw, exact, nil +} + +func byteOffsetForColumn(line []byte, column int) int { + offset := 0 + for current := 1; current < column && offset < len(line); current++ { + _, width := utf8.DecodeRune(line[offset:]) + offset += width + } + return offset +} + +// yamlQuotedEnd returns the length of the quoted scalar at the start of text, +// and whether its closing quote is on this line. A quoted scalar that +// continues on the next line has no exact span. +func yamlQuotedEnd(text []byte) (int, bool) { + quote := text[0] + for index := 1; index < len(text); index++ { + switch { + case quote == '"' && text[index] == '\\': + index++ + case text[index] == quote && quote == '\'' && index+1 < len(text) && text[index+1] == '\'': + index++ + case text[index] == quote: + return index + 1, true + } + } + return len(text), false +} + +// yamlPlainScalar returns the plain scalar at the start of text. A space and +// a hash start a comment. In a flow collection, a comma or a closing bracket +// ends the scalar. +func yamlPlainScalar(text []byte, flow bool) []byte { + end := len(text) + if comment := bytes.Index(text, []byte(" #")); comment >= 0 { + end = comment + } + if flow { + if index := bytes.IndexAny(text[:end], ",]}"); index >= 0 { + end = index + } + } + return bytes.TrimRight(text[:end], " \t") +} + +func quoteYAML(value, previous string) string { + if strings.HasPrefix(previous, "'") { + if strings.ContainsAny(value, "\r\n") { + encoded, _ := json.Marshal(value) + return string(encoded) + } + return "'" + strings.ReplaceAll(value, "'", "''") + "'" + } + if strings.HasPrefix(previous, `"`) { + encoded, _ := json.Marshal(value) + return string(encoded) + } + if value != "" && !strings.ContainsAny(value, ":\n\r#{}[],&*!|>'\"%@`") && + value == strings.TrimSpace(value) { + var decoded any + if yaml.Unmarshal([]byte(value), &decoded) == nil { + if _, isString := decoded.(string); isString { + return value + } + } + } + encoded, _ := json.Marshal(value) + return string(encoded) +} + +// EncodeString writes a string in the quoting style of the scalar that it +// replaces. It rejects characters that YAML cannot hold. +func EncodeString(value, previous string) ([]byte, error) { + for _, character := range value { + valid := character == '\n' || character == '\r' || character == '\t' || + character >= 0x20 && character <= 0xd7ff || + character >= 0xe000 && character <= 0xfffd || + character >= 0x10000 && character <= utf8.MaxRune + if !valid || unicode.IsControl(character) && character != '\n' && character != '\r' && character != '\t' { + return nil, fmt.Errorf("YAML string contains unsupported character U+%04X", character) + } + } + return []byte(quoteYAML(value, previous)), nil +} + +// Build builds the logical node of one YAML node. flow is true +// inside a flow sequence or a flow mapping. +func Build(data []byte, node *yaml.Node, offset int, flow bool) (*adapter.Node, error) { + return build(data, node, offset, flow, map[*yaml.Node]bool{}) +} + +// build is Build with the set of anchored nodes that are still being built. +// An alias to one of those nodes is a cycle, which has no finite value. +func build(data []byte, node *yaml.Node, offset int, flow bool, open map[*yaml.Node]bool) (*adapter.Node, error) { + if node.Anchor != "" { + if open[node] { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + fmt.Sprintf("YAML alias *%s refers to a node that contains it", node.Anchor), + "Remove the circular alias.", + ) + } + open[node] = true + defer delete(open, node) + } + flow = flow || node.Style&yaml.FlowStyle != 0 + switch node.Kind { + case yaml.MappingNode: + return yamlMappingNode(data, node, offset, flow, open) + case yaml.SequenceNode: + return yamlSequenceNode(data, node, offset, flow, open) + case yaml.ScalarNode: + return yamlValueNode(data, node, offset, flow) + case yaml.AliasNode: + value, err := build(data, node.Alias, offset, flow, open) + if value != nil { + value.MarkReadOnly("YAML aliases are read-only") + } + return value, err + default: + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML node is not supported", "Use mappings, lists, or scalar values.", + ) + } +} + +func yamlMappingNode(data []byte, node *yaml.Node, offset int, flow bool, open map[*yaml.Node]bool) (*adapter.Node, error) { + fields := make(map[string]*adapter.Node, len(node.Content)/2) + value := make(map[string]any, len(node.Content)/2) + unsafe := node.Anchor != "" || customYAMLTag(node.Tag) + reason := "YAML anchors and tags are read-only" + var merges []*adapter.Node + merged := false + for index := 0; index+1 < len(node.Content); index += 2 { + keyNode, valueNode := node.Content[index], node.Content[index+1] + if keyNode.Kind != yaml.ScalarNode || keyNode.Tag != "!!str" && keyNode.Tag != "!!merge" { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "YAML mapping key is not a string", + "Use direct string keys.", + ) + } + key := keyNode.Value + if _, duplicate := fields[key]; duplicate || key == "<<" && merged { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("YAML mapping contains duplicate key %q", key), + "Remove the duplicate key.", + ) + } + child, err := build(data, valueNode, offset, flow, open) + if err != nil { + return nil, err + } + if key == "<<" { + unsafe, reason, merged = true, "YAML merge keys are read-only", true + switch { + case child.Fields != nil: + merges = []*adapter.Node{child} + case child.Collection: + merges = child.Items + default: + return nil, invalidMerge() + } + continue + } + fields[key], value[key] = child, child.Value + } + // Merged keys come last: an explicit key wins over a merged key, and an + // earlier mapping in a merge list wins over a later one. + for _, merged := range merges { + if merged.Fields == nil { + return nil, invalidMerge() + } + for mergedKey, mergedValue := range merged.Fields { + if _, exists := fields[mergedKey]; !exists { + fields[mergedKey] = mergedValue + value[mergedKey] = mergedValue.Value + } + } + } + result := &adapter.Node{ + Value: value, Fields: fields, Capability: adapter.CapabilityReadOnly, + Reason: "YAML structural changes are read-only", + } + if unsafe { + result.MarkReadOnly(reason) + } + return result, nil +} + +func invalidMerge() error { + return syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "YAML merge key value is not a mapping or a list of mappings", + "Merge one mapping or a list of mappings.", + ) +} + +func yamlSequenceNode(data []byte, node *yaml.Node, offset int, flow bool, open map[*yaml.Node]bool) (*adapter.Node, error) { + items := make([]*adapter.Node, 0, len(node.Content)) + values := make([]any, 0, len(node.Content)) + for _, childNode := range node.Content { + child, err := build(data, childNode, offset, flow, open) + if err != nil { + return nil, err + } + items, values = append(items, child), append(values, child.Value) + } + result := &adapter.Node{ + Value: values, Items: items, Collection: true, Capability: adapter.CapabilityReadOnly, + Reason: "YAML structural changes are read-only", + } + if node.Anchor != "" || customYAMLTag(node.Tag) { + result.MarkReadOnly("YAML anchors and tags are read-only") + } + return result, nil +} + +func yamlValueNode(data []byte, node *yaml.Node, offset int, flow bool) (*adapter.Node, error) { + region := Region{Data: data, StartOffset: offset} + start, end, raw, exact, err := ScalarSpan(region, node, flow) + if err != nil { + return nil, err + } + readOnly := !exact || node.Anchor != "" || customYAMLTag(node.Tag) || + node.Style == yaml.LiteralStyle || node.Style == yaml.FoldedStyle + reason := "YAML anchors, tags, block scalars, and multi-line scalars are read-only" + var result *adapter.Node + switch node.Tag { + case "!!str": + result = adapter.String(node.Value, start, end, func(text string) ([]byte, error) { + return EncodeString(text, raw) + }) + case "!!bool": + var value bool + if err := node.Decode(&value); err != nil { + result = unreadableScalar(node, start, end) + break + } + result = adapter.Bool(value, start, end, nil) + case "!!int", "!!float": + var number float64 + if err := node.Decode(&number); err != nil || math.IsInf(number, 0) || math.IsNaN(number) { + result = unreadableScalar(node, start, end) + break + } + result = adapter.Number(number, start, end, func(next float64) ([]byte, error) { + return []byte(strconv.FormatFloat(next, 'g', -1, 64)), nil + }) + case "!!null": + result = adapter.ReadOnly(start, end, "YAML null values are read-only") + default: + result = adapter.ReadOnly(start, end, "YAML scalar type is read-only") + result.Value = node.Value + } + if readOnly { + result.Capability, result.Reason, result.Encode = adapter.CapabilityReadOnly, reason, nil + } + return result, nil +} + +// unreadableScalar keeps a scalar that the YAML library cannot decode, or +// that has no JSON form such as infinity, as a read-only value, so that one +// odd spelling does not reject the document. +func unreadableScalar(node *yaml.Node, start, end int) *adapter.Node { + result := adapter.ReadOnly(start, end, "YAML scalar spelling is read-only") + result.Value = node.Value + return result +} + +func customYAMLTag(tag string) bool { + return tag != "" && !strings.HasPrefix(tag, "!!") +} diff --git a/internal/sync/adapter/yamltree/yamltree_test.go b/internal/sync/adapter/yamltree/yamltree_test.go new file mode 100644 index 000000000..8c1f5c6f5 --- /dev/null +++ b/internal/sync/adapter/yamltree/yamltree_test.go @@ -0,0 +1,140 @@ +package yamltree + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +func TestScalarSpanHelpers(t *testing.T) { + _, _, _, _, err := ScalarSpan(Region{}, &yaml.Node{}, false) + require.Error(t, err) + assert.Equal(t, `"line\nnext"`, quoteYAML("line\nnext", "'old'")) + assert.Equal(t, "'it''s'", quoteYAML("it's", "'old'")) + _, err = EncodeString("bad\x01", "old") + require.Error(t, err) + + end, closed := yamlQuotedEnd([]byte(`"unterminated`)) + assert.Equal(t, len(`"unterminated`), end) + assert.False(t, closed) + end, closed = yamlQuotedEnd([]byte(`'it''s'`)) + assert.Equal(t, len(`'it''s'`), end) + assert.True(t, closed) + assert.Equal(t, "value, next", string(yamlPlainScalar([]byte("value, next # comment"), false))) + assert.Equal(t, "value", string(yamlPlainScalar([]byte("value, next]"), true))) +} + +func TestQuotedScalarsThatSpanLinesAreReadOnly(t *testing.T) { + data := []byte("prompt: \"first line\n second line\"\nnext: value\n") + var document yaml.Node + require.NoError(t, yaml.Unmarshal(data, &document)) + root, err := Build(data, document.Content[0], 0, false) + require.NoError(t, err) + + prompt := root.Fields["prompt"] + assert.Equal(t, "first line second line", prompt.Value) + assert.Equal(t, adapter.CapabilityReadOnly, prompt.Capability) + assert.Equal(t, adapter.CapabilityDirect, root.Fields["next"].Capability) +} + +func TestNumberSpellingsDecodeOrStayReadOnly(t *testing.T) { + data := []byte("hex: 0x1A\noctal: 0o17\nseparated: 1_000\nflag: True\n") + var document yaml.Node + require.NoError(t, yaml.Unmarshal(data, &document)) + root, err := Build(data, document.Content[0], 0, false) + require.NoError(t, err) + + assert.Equal(t, 26.0, root.Fields["hex"].Value) + assert.Equal(t, 15.0, root.Fields["octal"].Value) + assert.Equal(t, 1000.0, root.Fields["separated"].Value) + assert.Equal(t, true, root.Fields["flag"].Value) + assert.Equal(t, adapter.CapabilityReadOnly, unreadableScalar(&yaml.Node{Value: "odd"}, 0, 3).Capability) +} + +func TestValidYAMLNumbersDoNotFailTheDocument(t *testing.T) { + for _, value := range []string{"0x10", "0o17", ".inf", "-.inf", ".nan"} { + t.Run(value, func(t *testing.T) { + source := "limit: " + value + "\nname: kept\n" + var document yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(source), &document)) + root, err := Build([]byte(source), document.Content[0], 0, false) + require.NoError(t, err) + assert.Equal(t, "kept", root.Fields["name"].Value) + limit := root.Fields["limit"] + if _, finite := limit.Value.(float64); finite { + assert.Equal(t, adapter.CapabilityDirect, limit.Capability) + return + } + // Infinity and NaN have no JSON form, so they stay read-only text. + assert.Equal(t, value, limit.Value) + assert.Equal(t, adapter.CapabilityReadOnly, limit.Capability) + }) + } +} + +func TestCircularAliasIsRejected(t *testing.T) { + data := []byte("self: &self\n again: *self\n") + var document yaml.Node + require.NoError(t, yaml.Unmarshal(data, &document)) + _, err := Build(data, document.Content[0], 0, false) + require.Error(t, err) + assert.Contains(t, err.Error(), "*self refers to a node that contains it") + + data = []byte("base: &base\n name: shared\ncopy: *base\nlist: [*base, *base]\n") + document = yaml.Node{} + require.NoError(t, yaml.Unmarshal(data, &document)) + root, err := Build(data, document.Content[0], 0, false) + require.NoError(t, err) + assert.Equal(t, "shared", root.Fields["copy"].Fields["name"].Value) + assert.Len(t, root.Fields["list"].Items, 2) +} + +func TestNullScalarsHaveNoValue(t *testing.T) { + data := []byte("tilde: ~\nword: null\nempty:\n") + var document yaml.Node + require.NoError(t, yaml.Unmarshal(data, &document)) + root, err := Build(data, document.Content[0], 0, false) + require.NoError(t, err) + for _, key := range []string{"tilde", "word", "empty"} { + assert.Nil(t, root.Fields[key].Value, key) + assert.Equal(t, adapter.CapabilityReadOnly, root.Fields[key].Capability, key) + } +} + +func TestMergeKeysLetExplicitKeysWin(t *testing.T) { + data := []byte("base: &base\n name: shared\n limit: 1\nother: &other\n limit: 2\n extra: true\nagent:\n <<: *base\n name: own\nlist:\n limit: 9\n <<: [*base, *other]\n") + var document yaml.Node + require.NoError(t, yaml.Unmarshal(data, &document)) + root, err := Build(data, document.Content[0], 0, false) + require.NoError(t, err) + + agent := root.Fields["agent"] + assert.Equal(t, "own", agent.Fields["name"].Value) + assert.Equal(t, 1.0, agent.Fields["limit"].Value) + assert.NotContains(t, agent.Fields, "<<") + assert.Equal(t, adapter.CapabilityReadOnly, agent.Capability) + + list := root.Fields["list"] + assert.Equal(t, 9.0, list.Fields["limit"].Value) + assert.Equal(t, "shared", list.Fields["name"].Value) + assert.Equal(t, true, list.Fields["extra"].Value) + assert.Equal(t, map[string]any{"limit": 9.0, "name": "shared", "extra": true}, list.Value) + + for name, source := range map[string]string{ + "duplicate key": "a: 1\na: 2\n", + "duplicate merge": "b: &b\n x: 1\nc:\n <<: *b\n <<: *b\n", + "scalar merge": "c:\n <<: text\n", + "scalar in list": "b: &b\n x: 1\nc:\n <<: [*b, text]\n", + } { + t.Run(name, func(t *testing.T) { + var document yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(source), &document)) + _, err := Build([]byte(source), document.Content[0], 0, false) + require.Error(t, err) + }) + } +}