From bfe91d038683751845db6bf848c6458fe34f3aee Mon Sep 17 00:00:00 2001 From: Cliff Tawiah Date: Fri, 30 Oct 2026 16:43:43 -0400 Subject: [PATCH 1/5] refactor(sync): remove Phase 1 source references --- cmd/sync/prompt.go | 55 ---- cmd/sync/prompt_test.go | 10 +- cmd/sync/scope_test.go | 25 +- internal/sync/detach/detach_test.go | 43 --- internal/sync/link/link.go | 310 ------------------ internal/sync/link/link_test.go | 279 ---------------- internal/sync/local/compile.go | 14 +- internal/sync/local/reference.go | 120 ------- internal/sync/local/reference_test.go | 185 ----------- internal/sync/local/replace.go | 54 +-- internal/sync/local/sources.go | 27 +- internal/sync/local/store.go | 6 +- internal/sync/local/variation.go | 44 +-- internal/sync/manifest/baseline_test.go | 22 -- internal/sync/manifest/model.go | 21 -- internal/sync/prompt/acceptance_test.go | 150 --------- internal/sync/prompt/action.go | 30 -- internal/sync/prompt/execute.go | 9 - internal/sync/prompt/plan.go | 2 - internal/sync/prompt/runner.go | 32 -- internal/sync/prompt/runner_test.go | 30 -- internal/sync/prompt/state.go | 40 --- internal/sync/prompt/watch_test.go | 146 --------- internal/sync/reference/adapters/adapter.go | 20 -- .../adapters/plainmarkdown/plainmarkdown.go | 39 --- .../plainmarkdown/plainmarkdown_test.go | 45 --- internal/sync/reference/format.go | 119 ------- internal/sync/reference/format_test.go | 79 ----- internal/sync/resource.go | 4 +- 29 files changed, 35 insertions(+), 1925 deletions(-) delete mode 100644 internal/sync/link/link.go delete mode 100644 internal/sync/link/link_test.go delete mode 100644 internal/sync/local/reference.go delete mode 100644 internal/sync/local/reference_test.go delete mode 100644 internal/sync/reference/adapters/adapter.go delete mode 100644 internal/sync/reference/adapters/plainmarkdown/plainmarkdown.go delete mode 100644 internal/sync/reference/adapters/plainmarkdown/plainmarkdown_test.go delete mode 100644 internal/sync/reference/format.go delete mode 100644 internal/sync/reference/format_test.go diff --git a/cmd/sync/prompt.go b/cmd/sync/prompt.go index 9b69806be..34d73fb2f 100644 --- a/cmd/sync/prompt.go +++ b/cmd/sync/prompt.go @@ -15,7 +15,6 @@ import ( "github.com/launchdarkly/ldcli/internal/output" "github.com/launchdarkly/ldcli/internal/resources" syncdomain "github.com/launchdarkly/ldcli/internal/sync" - synclink "github.com/launchdarkly/ldcli/internal/sync/link" syncprompt "github.com/launchdarkly/ldcli/internal/sync/prompt" ) @@ -68,7 +67,6 @@ func NewPromptCmd(client resources.Client) *cobra.Command { newAddCmd(client), newAttachCmd(client), newDetachCmd(client), - newLinkCmd(client), ) cmd.SetUsageTemplate(resourcescmd.SubcommandUsageTemplate()) @@ -270,37 +268,6 @@ func newDetachCmd(client resources.Client) *cobra.Command { return cmd } -func newLinkCmd(client resources.Client) *cobra.Command { - cmd := &cobra.Command{ - Use: "link ", - Short: "Link an external prompt file", - Example: ` # Link a Markdown file as a new variation - ldcli sync prompts link prompts/support.md \ - --format plain-markdown \ - --to production/support/default \ - --model-config-key claude \ - --yes --no-input`, - Args: validatedArgs(cobra.ExactArgs(1)), - RunE: runCommand(client, syncdomain.ScopePrompts, func(cmd *cobra.Command, args []string) (syncprompt.CommandAction, error) { - format, _ := cmd.Flags().GetString(formatFlag) - target, err := linkTarget(cmd) - if err != nil { - return nil, err - } - return syncprompt.LinkAction{File: args[0], Format: format, ProjectKey: targetFilter(cmd).Project, Target: target}, nil - }), - } - addTargetFlags(cmd, targetFlagHelp{project: questionFlagHelp.project}) - cmd.Flags().String(formatFlag, "", "Format adapter (for example, plain-markdown)") - cmd.Flags().String(toFlag, "", "New variation (project-key/config-key/variation-key)") - cmd.Flags().String(modelConfigKeyFlag, "", "Model config key for the new variation") - cmd.Flags().String(nameFlag, "", "Variation name when absent from the linked file") - cmd.Flags().String(contentFlag, "", "Prompt content when absent from the linked file") - _ = cmd.MarkFlagRequired(formatFlag) - addApplyFlags(cmd, syncdomain.ScopePrompts, false) - return cmd -} - // addApplyFlags registers the flags of a command that applies a plan. Only // the prompts scope has --resolve, which names variations. func addApplyFlags(cmd *cobra.Command, scope syncdomain.Scope, dryRun bool) { @@ -434,28 +401,6 @@ func optionalVariationFlag(cmd *cobra.Command, name string) (*syncdomain.Resourc return &variation, nil } -func linkTarget(cmd *cobra.Command) (*synclink.Target, error) { - variation, err := optionalVariationFlag(cmd, toFlag) - if err != nil { - return nil, err - } - modelConfig, _ := cmd.Flags().GetString(modelConfigKeyFlag) - name, _ := cmd.Flags().GetString(nameFlag) - content, _ := cmd.Flags().GetString(contentFlag) - if variation == nil { - if modelConfig != "" || name != "" || content != "" { - return nil, fmt.Errorf("--model-config-key, --name, and --content require --to") - } - return nil, nil - } - return &synclink.Target{ - Variation: *variation, - ModelConfigKey: modelConfig, - Name: name, - Content: content, - }, nil -} - func conflictPolicy(cmd *cobra.Command) (syncprompt.ConflictPolicy, error) { var policy syncprompt.ConflictPolicy if cmd.Flags().Lookup(conflictFlag) == nil { diff --git a/cmd/sync/prompt_test.go b/cmd/sync/prompt_test.go index fc7dbc954..f3708811c 100644 --- a/cmd/sync/prompt_test.go +++ b/cmd/sync/prompt_test.go @@ -17,21 +17,17 @@ func TestPromptCommandUsesActionSubcommands(t *testing.T) { command := NewPromptCmd(nil) assert.Equal(t, "prompts", command.Use) - for _, name := range []string{"watch", "add", "attach", "detach", "link"} { + for _, name := range []string{"watch", "add", "attach", "detach"} { child, _, err := command.Find([]string{name}) require.NoError(t, err) assert.Equal(t, name, child.Name()) } - for _, name := range []string{"add", "attach-tool", "attach-skill", "detach", "link", "watch"} { + for _, name := range []string{"add", "attach-tool", "attach-skill", "detach", "watch"} { assert.Nil(t, command.Flags().Lookup(name), "legacy --%s should not exist", name) } add, _, err := command.Find([]string{"add"}) require.NoError(t, err) assert.Equal(t, "add [project-key/config-key/variation-key...]", add.Use) - link, _, err := command.Find([]string{"link"}) - require.NoError(t, err) - assert.NotNil(t, link.Flags().Lookup(modelConfigKeyFlag)) - assert.Nil(t, link.Flags().Lookup("model-config")) } func TestSyncCommandIncludesExamples(t *testing.T) { @@ -57,7 +53,6 @@ func TestPromptCommandHelpIncludesExamples(t *testing.T) { "attach tool": {"attach", "tool"}, "attach skill": {"attach", "skill"}, "detach": {"detach"}, - "link": {"link"}, } for name, path := range tests { @@ -112,7 +107,6 @@ func TestPromptCommandsRegisterTargetFlags(t *testing.T) { "add": {path: []string{"add"}, help: questionFlagHelp}, "detach": {path: []string{"detach"}, help: questionFlagHelp}, "attach tool": {path: []string{"attach", "tool"}, help: targetFlagHelp{project: questionFlagHelp.project}}, - "link": {path: []string{"link"}, help: targetFlagHelp{project: questionFlagHelp.project}}, } for name, test := range tests { diff --git a/cmd/sync/scope_test.go b/cmd/sync/scope_test.go index a75d894cd..1d7a0ec9a 100644 --- a/cmd/sync/scope_test.go +++ b/cmd/sync/scope_test.go @@ -25,17 +25,14 @@ var syncCommandFlags = map[string][]string{ "sync prompts attach tool": {"conflict", "diff", "no-input", "project", "resolve", "to", "yes"}, "sync prompts attach skill": {"conflict", "diff", "no-input", "project", "resolve", "to", "yes"}, "sync prompts detach": {"archive", "config-key", "key", "no-input", "project", "yes"}, - "sync prompts link": { - "conflict", "content", "diff", "format", "model-config-key", "name", "no-input", "project", "resolve", "to", "yes", - }, - "sync tools": {"conflict", "diff", "dry-run", "key", "no-input", "project", "yes"}, - "sync tools watch": {"conflict", "diff", "key", "no-input", "project", "yes"}, - "sync tools add": {"description", "diff", "dry-run", "json-schema", "key", "no-input", "project", "yes"}, - "sync tools detach": {"key", "no-input", "project", "yes"}, - "sync skills": {"conflict", "diff", "dry-run", "key", "no-input", "project", "yes"}, - "sync skills watch": {"conflict", "diff", "key", "no-input", "project", "yes"}, - "sync skills add": {"content", "description", "diff", "dry-run", "key", "name", "no-input", "project", "yes"}, - "sync skills detach": {"key", "no-input", "project", "yes"}, + "sync tools": {"conflict", "diff", "dry-run", "key", "no-input", "project", "yes"}, + "sync tools watch": {"conflict", "diff", "key", "no-input", "project", "yes"}, + "sync tools add": {"description", "diff", "dry-run", "json-schema", "key", "no-input", "project", "yes"}, + "sync tools detach": {"key", "no-input", "project", "yes"}, + "sync skills": {"conflict", "diff", "dry-run", "key", "no-input", "project", "yes"}, + "sync skills watch": {"conflict", "diff", "key", "no-input", "project", "yes"}, + "sync skills add": {"content", "description", "diff", "dry-run", "key", "name", "no-input", "project", "yes"}, + "sync skills detach": {"key", "no-input", "project", "yes"}, } func TestSyncCommandsHaveExactFlags(t *testing.T) { @@ -75,11 +72,7 @@ func TestSyncCommandsReadSyncFlagsOnlyFromTheCommandLine(t *testing.T) { for _, command := range runnableSyncCommands(NewSyncCmd(nil, nil)) { require.NoError(t, command.ParseFlags(nil)) - var args []string - if command.Name() == "link" { - args = []string{"prompt.md"} - } - require.NoError(t, command.Args(command, args), command.CommandPath()) + require.NoError(t, command.Args(command, nil), command.CommandPath()) command.LocalNonPersistentFlags().VisitAll(func(flag *pflag.Flag) { assert.False(t, flag.Changed, "%s --%s", command.CommandPath(), flag.Name) }) diff --git a/internal/sync/detach/detach_test.go b/internal/sync/detach/detach_test.go index d7865a1e1..9c6aa7427 100644 --- a/internal/sync/detach/detach_test.go +++ b/internal/sync/detach/detach_test.go @@ -18,7 +18,6 @@ import ( syncapi "github.com/launchdarkly/ldcli/internal/sync/api" synclocal "github.com/launchdarkly/ldcli/internal/sync/local" syncmanifest "github.com/launchdarkly/ldcli/internal/sync/manifest" - syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" ) func TestLoadResourcesUnionsLocalAndManifestResources(t *testing.T) { @@ -97,48 +96,6 @@ func TestDetachResourcesKeepsAttachmentManifestEntries(t *testing.T) { require.NoError(t, err, "the tool file stays") } -func TestDetachResourcesRemovesWrapperAndManifestButKeepsReferencedFile(t *testing.T) { - root := t.TempDir() - referencePath := filepath.Join(root, "prompts", "prompt.md") - require.NoError(t, os.MkdirAll(filepath.Dir(referencePath), 0o755)) - require.NoError(t, os.WriteFile(referencePath, []byte("Keep me.\n"), 0o644)) - - store := synclocal.NewStore(root) - variation := testVariation("prompt") - _, err := store.Add([]synclocal.VariationFile{{ - ProjectKey: "project", ConfigKey: "config", Variation: variation, - Ref: &synclocal.Reference{File: "prompts/prompt.md", Format: syncreference.PlainMarkdown}, - }}) - require.NoError(t, err) - - manifestStore := newMemoryManifestStore() - original := syncmanifest.Manifest{ - Resources: []syncmanifest.Resource{{ - ResourceKind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/prompt", - Fingerprint: testFingerprint(), - }}, - } - require.NoError(t, manifestStore.Write(original)) - - resource := syncdomain.ResourceID{Kind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/prompt"} - err = detachResources( - Options{RepositoryRoot: root, Store: store, Baselines: manifestStore}, - syncmanifest.Baseline{Lock: original}, - []syncdomain.ResourceID{resource}, - ) - - require.NoError(t, err) - exists, err := store.VariationExists("project", "config", "prompt") - require.NoError(t, err) - assert.False(t, exists) - _, err = os.Stat(referencePath) - require.NoError(t, err) - baseline, err := manifestStore.Load([]string{"project"}) - manifest := baseline.Lock - require.NoError(t, err) - assert.Empty(t, manifest.Resources) -} - func TestDetachResourcesRemovesManifestEntryWhenWrapperWasAlreadyDeleted(t *testing.T) { root := t.TempDir() store := synclocal.NewStore(root) diff --git a/internal/sync/link/link.go b/internal/sync/link/link.go deleted file mode 100644 index bbf6ed760..000000000 --- a/internal/sync/link/link.go +++ /dev/null @@ -1,310 +0,0 @@ -// Package link creates a variation whose prompt lives in an external file in -// the repository. The variation file stores a reference to that file. -package link - -import ( - "bytes" - "cmp" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "slices" - "strings" - - "github.com/charmbracelet/huh" - - syncdomain "github.com/launchdarkly/ldcli/internal/sync" - syncapi "github.com/launchdarkly/ldcli/internal/sync/api" - synccreate "github.com/launchdarkly/ldcli/internal/sync/create" - syncinteractive "github.com/launchdarkly/ldcli/internal/sync/interactive" - synclocal "github.com/launchdarkly/ldcli/internal/sync/local" - syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" - "github.com/launchdarkly/ldcli/internal/sync/reference/adapters" -) - -// Options are the dependencies and the input of one link. -type Options struct { - Catalog synccreate.Catalog - Store synclocal.Store - RepositoryRoot string - WorkingDirectory string - File string - Format string - Input io.Reader - Output io.Writer - // ProjectKey skips the project question when Run asks for the destination. - ProjectKey string - // Target is the new variation. If it is nil, Run asks the user. - Target *Target - NoInput bool -} - -// Target is a new variation that the user named with flags. Name and Content -// replace values that the linked file does not have. -type Target struct { - Variation syncdomain.ResourceID - ModelConfigKey string - Name string - Content string -} - -// Selection is the complete destination of a new linked variation. -type Selection struct { - Project syncapi.Project - Config syncapi.Config - ModelConfig syncapi.ModelConfig - Key string - Name string -} - -// linkedPrompt is the external file. If the file has no prompt, content is -// the new prompt that link writes to the file. -type linkedPrompt struct { - reference synclocal.Reference - parsed adapters.Prompt - originalContent []byte - content []byte -} - -// Run creates the linked variation file and returns its path. The path is -// empty when the user cancels. -func Run(options Options) (string, error) { - prompt, err := readLinkedPrompt(options) - if err != nil { - return "", err - } - - var selection Selection - var content string - if options.Target != nil { - selection, content, err = targetSelection(options, prompt, *options.Target) - } else { - if err := syncinteractive.RequireTerminal( - options.Input, options.Output, options.NoInput, "--to and --model-config-key", "prompt linking", - ); err != nil { - return "", err - } - var canceled bool - selection, content, canceled, err = promptForSelection(options, prompt) - if canceled { - return "", nil - } - } - if err != nil { - return "", err - } - - prompt, err = addMissingPromptContent(prompt, selection.Config.Mode, selection.Key, selection.Name, content) - if err != nil { - return "", err - } - return createLinkedPrompt(options, selection, prompt) -} - -// targetSelection completes the destination that the user named with flags. -// It returns the prompt content to add when the linked file has none. -func targetSelection(options Options, prompt linkedPrompt, target Target) (Selection, string, error) { - configKey, variationKey, err := target.Variation.VariationKeys() - if err != nil { - return Selection{}, "", fmt.Errorf("link destination must be a variation: %w", err) - } - hasContent := len(prompt.parsed.Messages) != 0 - switch { - case prompt.parsed.Key != "" && prompt.parsed.Key != variationKey: - return Selection{}, "", fmt.Errorf("linked file key %q does not match destination key %q", prompt.parsed.Key, variationKey) - case target.Name != "" && strings.TrimSpace(target.Name) == "": - return Selection{}, "", errors.New("--name cannot be blank") - case target.Content != "" && strings.TrimSpace(target.Content) == "": - return Selection{}, "", errors.New("--content cannot be blank") - case prompt.parsed.Name != "" && target.Name != "" && prompt.parsed.Name != target.Name: - return Selection{}, "", fmt.Errorf("linked file name %q does not match --name %q", prompt.parsed.Name, target.Name) - case !hasContent && target.Content == "": - return Selection{}, "", errors.New("--content is required when the linked file has no prompt content") - case hasContent && target.Content != "": - return Selection{}, "", errors.New("--content cannot be used when the linked file already has prompt content") - } - - projectKey := target.Variation.ProjectKey - config, err := options.Catalog.Config(projectKey, configKey) - if err != nil { - return Selection{}, "", err - } - modelConfig, err := synccreate.FindModelConfig(options.Catalog, projectKey, target.ModelConfigKey) - if err != nil { - return Selection{}, "", err - } - - name := cmp.Or(prompt.parsed.Name, target.Name, syncdomain.DisplayName(variationKey)) - return Selection{ - Project: syncapi.Project{Key: projectKey}, - Config: config, - ModelConfig: modelConfig, - Key: variationKey, - Name: name, - }, target.Content, nil -} - -// promptForSelection asks the user for the destination, and for each value -// that the linked file does not have. The bool result is true when the user -// cancels. -func promptForSelection(options Options, prompt linkedPrompt) (Selection, string, bool, error) { - questions := synccreate.Questions{Allowed: true, Input: options.Input, Output: options.Output} - var modes []syncdomain.VariationMode - if prompt.parsed.Mode != "" { - modes = []syncdomain.VariationMode{prompt.parsed.Mode} - } - project, config, canceled, err := synccreate.SelectConfig(questions, options.Catalog, options.ProjectKey, modes) - if err != nil || canceled { - return Selection{}, "", canceled, err - } - modelConfig, canceled, err := synccreate.SelectModelConfig(questions, options.Catalog, project.Key) - if err != nil || canceled { - return Selection{}, "", canceled, err - } - - // Ask only for the values that the linked file does not have. - key, name, content := prompt.parsed.Key, prompt.parsed.Name, "" - var fields []huh.Field - if key == "" { - key = strings.TrimSuffix(filepath.Base(prompt.reference.File), filepath.Ext(prompt.reference.File)) - fields = append(fields, huh.NewInput().Title("Variation key").Value(&key).Validate(validateDerivedKey)) - } - if name == "" { - name = syncdomain.DisplayName(key) - fields = append(fields, huh.NewInput().Title("Variation name").Value(&name).Validate(synccreate.RequiredValue("variation name"))) - } - if len(prompt.parsed.Messages) == 0 { - fields = append(fields, huh.NewText().Title("Prompt content").Lines(8).Value(&content). - Validate(synccreate.RequiredValue("prompt content"))) - } - if len(fields) != 0 { - if canceled, err := syncinteractive.RunForm(options.Input, options.Output, fields...); err != nil || canceled { - return Selection{}, "", canceled, err - } - } - - return Selection{Project: project, Config: config, ModelConfig: modelConfig, Key: key, Name: name}, content, false, nil -} - -// addMissingPromptContent renders content as the new text of a linked file -// that has no prompt. It does not write the file. createLinkedPrompt writes -// it after every check passes. -func addMissingPromptContent( - prompt linkedPrompt, - mode syncdomain.VariationMode, - key, name, content string, -) (linkedPrompt, error) { - if len(prompt.parsed.Messages) != 0 { - return prompt, nil - } - variation := syncdomain.Variation{Mode: mode, Key: key, Name: name} - if mode == syncdomain.VariationModeAgent { - variation.Instructions = content - } else { - variation.Messages = []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: content}} - } - rendered, err := syncreference.Render(prompt.reference.Format, variation) - if err != nil { - return linkedPrompt{}, err - } - prompt.content = rendered - return prompt, nil -} - -// createLinkedPrompt checks the destination, writes new prompt content to the -// linked file, and creates the variation file. If the variation file fails, -// it restores the linked file. -func createLinkedPrompt(options Options, selection Selection, prompt linkedPrompt) (string, error) { - variation := synccreate.NewVariation(selection.Config, selection.Key, selection.Name, selection.ModelConfig) - if err := syncreference.ApplyToVariation(prompt.reference.Format, prompt.content, &variation); err != nil { - return "", err - } - - // Every check must pass before the linked file changes. - if variation.Mode != selection.Config.Mode { - return "", fmt.Errorf("referenced prompt mode %q does not match config mode %q", variation.Mode, selection.Config.Mode) - } - if err := validateDerivedKey(variation.Key); err != nil { - return "", err - } - if err := variation.Validate(); err != nil { - return "", fmt.Errorf("variation %q: %w", variation.Key, err) - } - if slices.ContainsFunc(selection.Config.Variations, func(existing syncdomain.Variation) bool { - return existing.Key == variation.Key - }) { - return "", fmt.Errorf("variation %q already exists in config %q", variation.Key, selection.Config.Key) - } - exists, err := options.Store.VariationExists(selection.Project.Key, selection.Config.Key, variation.Key) - if err != nil { - return "", err - } - if exists { - return "", fmt.Errorf("variation %q is already linked locally", variation.Key) - } - - source := filepath.Join(options.RepositoryRoot, filepath.FromSlash(prompt.reference.File)) - sourceChanged := !bytes.Equal(prompt.originalContent, prompt.content) - var sourceMode os.FileMode - if sourceChanged { - info, err := os.Stat(source) - if err != nil { - return "", err - } - sourceMode = info.Mode().Perm() - err = synclocal.ReplaceFileAtomically(source, prompt.reference.File, prompt.originalContent, prompt.content, sourceMode) - if err != nil { - return "", fmt.Errorf("write linked file %q: %w", prompt.reference.File, err) - } - } - - creation, err := options.Store.Add([]synclocal.VariationFile{{ - ProjectKey: selection.Project.Key, - ConfigKey: selection.Config.Key, - Upsert: true, - Ref: &prompt.reference, - Variation: variation, - }}) - if err != nil { - if sourceChanged { - restoreErr := synclocal.ReplaceFileAtomically( - source, prompt.reference.File, prompt.content, prompt.originalContent, sourceMode, - ) - if restoreErr != nil { - err = errors.Join(err, fmt.Errorf("restore linked file %q: %w", prompt.reference.File, restoreErr)) - } - } - return "", err - } - return creation.VariationPaths[0], nil -} - -// readLinkedPrompt reads and parses the external file. -func readLinkedPrompt(options Options) (linkedPrompt, error) { - reference, err := synclocal.NewReference(options.RepositoryRoot, options.WorkingDirectory, options.File, options.Format) - if err != nil { - return linkedPrompt{}, err - } - content, err := os.ReadFile(filepath.Join(options.RepositoryRoot, filepath.FromSlash(reference.File))) - if err != nil { - return linkedPrompt{}, fmt.Errorf("read linked file %q: %w", reference.File, err) - } - prompt, err := syncreference.Parse(reference.Format, content) - if err != nil { - return linkedPrompt{}, err - } - return linkedPrompt{reference: reference, parsed: prompt, originalContent: content, content: content}, nil -} - -// validateDerivedKey makes sure that a key from a file name is a safe key. -func validateDerivedKey(key string) error { - if key == "" { - return errors.New("cannot derive a variation key from the linked filename") - } - if syncdomain.ValidateKey(key) != nil { - return fmt.Errorf("linked filename produces invalid variation key %q", key) - } - return nil -} diff --git a/internal/sync/link/link_test.go b/internal/sync/link/link_test.go deleted file mode 100644 index 6705904d0..000000000 --- a/internal/sync/link/link_test.go +++ /dev/null @@ -1,279 +0,0 @@ -package link - -import ( - "bytes" - "errors" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" - - syncdomain "github.com/launchdarkly/ldcli/internal/sync" - syncapi "github.com/launchdarkly/ldcli/internal/sync/api" - synclocal "github.com/launchdarkly/ldcli/internal/sync/local" - syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" -) - -func TestRunUsesExplicitTargetWithoutTerminal(t *testing.T) { - root := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(root, "prompt.md"), []byte("Be helpful.\n"), 0o644)) - target := syncdomain.ResourceID{ - Kind: syncdomain.KindVariation, ProjectKey: "production", LookupKey: "support/support-agent", - } - - path, err := Run(Options{ - Catalog: &fakeCatalog{ - config: syncapi.Config{Key: "support", Mode: syncdomain.VariationModeAgent}, - model: syncapi.ModelConfig{Key: "claude", ID: "claude-3", Version: 2}, - }, - Store: synclocal.NewStore(root), - RepositoryRoot: root, - WorkingDirectory: root, - File: "prompt.md", - Format: syncreference.PlainMarkdown, - NoInput: true, - Target: &Target{ - Variation: target, ModelConfigKey: "claude", - }, - }) - - require.NoError(t, err) - require.Equal(t, "production/configs/support/support-agent.prompt.md", path) - wrapper, err := os.ReadFile(filepath.Join(root, syncdomain.RootDir, filepath.FromSlash(path))) - require.NoError(t, err) - require.Contains(t, string(wrapper), "name: Support agent") -} - -func TestRunRejectsBlankExplicitMetadata(t *testing.T) { - tests := map[string]Target{ - "name": { - Name: " ", - }, - "content": { - Content: "\t", - }, - } - - for field, target := range tests { - t.Run(field, func(t *testing.T) { - root := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(root, "prompt.md"), nil, 0o644)) - target.Variation = syncdomain.ResourceID{ - Kind: syncdomain.KindVariation, ProjectKey: "production", LookupKey: "support/support-agent", - } - target.ModelConfigKey = "claude" - - _, err := Run(Options{ - Catalog: &fakeCatalog{}, - Store: synclocal.NewStore(root), - RepositoryRoot: root, - WorkingDirectory: root, - File: "prompt.md", - Format: syncreference.PlainMarkdown, - Target: &target, - }) - - require.ErrorContains(t, err, "--"+field+" cannot be blank") - }) - } -} - -func TestCreateLinkedPromptWritesLinkedVariationWrapper(t *testing.T) { - root := t.TempDir() - require.NoError(t, os.Mkdir(filepath.Join(root, "prompts"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(root, "prompts", "support-agent.md"), []byte("Be helpful.\n"), 0o644)) - - path, err := create(Options{ - Store: synclocal.NewStore(root), - RepositoryRoot: root, - WorkingDirectory: root, - File: "prompts/support-agent.md", - Format: syncreference.PlainMarkdown, - }, Selection{ - Project: syncapi.Project{Key: "production", Name: "Production"}, - Config: syncapi.Config{Key: "support", Name: "Support", Mode: syncdomain.VariationModeAgent}, - ModelConfig: syncapi.ModelConfig{ - Key: "claude", ID: "claude-3-5-sonnet-20241022", Name: "Claude", Version: 4, - Params: map[string]any{"temperature": 0.2}, CustomParams: map[string]any{"region": "us-east"}, - }, - Key: "support-agent", - Name: "Support agent", - }) - - require.NoError(t, err) - require.Equal(t, "production/configs/support/support-agent.prompt.md", path) - wrapper, err := os.ReadFile(filepath.Join(root, syncdomain.RootDir, filepath.FromSlash(path))) - require.NoError(t, err) - require.Equal(t, `--- -formatVersion: 1 -upsert: true -ref: - file: prompts/support-agent.md - format: plain-markdown -mode: agent -key: support-agent -name: Support agent -modelConfigKey: claude -modelConfigVersion: 4 -model: - custom: - region: us-east - modelName: claude-3-5-sonnet-20241022 - parameters: - temperature: 0.2 ---- -`, string(wrapper)) - - resources, err := synclocal.CompileWorkspace(root) - require.NoError(t, err) - require.Len(t, resources, 1) - require.Equal(t, "Be helpful.", resources[0].Variation.Instructions) - require.Equal(t, "claude-3-5-sonnet-20241022", resources[0].Variation.Model["modelName"]) -} - -func TestCreateLinkedPromptRejectsExistingServerVariation(t *testing.T) { - root := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(root, "prompt.md"), []byte("Help"), 0o644)) - - _, err := create(Options{ - Store: synclocal.NewStore(root), - RepositoryRoot: root, - WorkingDirectory: root, - File: "prompt.md", - Format: syncreference.PlainMarkdown, - }, Selection{ - Project: syncapi.Project{Key: "production"}, - Config: syncapi.Config{ - Key: "support", Mode: syncdomain.VariationModeAgent, - Variations: []syncdomain.Variation{{Key: "prompt"}}, - }, - ModelConfig: syncapi.ModelConfig{Key: "claude", ID: "claude-3-5-sonnet-20241022"}, - Key: "prompt", - Name: "Prompt", - }) - - require.ErrorContains(t, err, `variation "prompt" already exists`) -} - -func TestCreateLinkedPromptValidatesDestinationBeforeUpdatingLinkedFile(t *testing.T) { - root := t.TempDir() - sourcePath := filepath.Join(root, "prompt.md") - require.NoError(t, os.WriteFile(sourcePath, nil, 0o644)) - options := Options{ - Store: synclocal.NewStore(root), - RepositoryRoot: root, - WorkingDirectory: root, - File: "prompt.md", - Format: syncreference.PlainMarkdown, - } - prompt, err := readLinkedPrompt(options) - require.NoError(t, err) - prompt, err = addMissingPromptContent(prompt, syncdomain.VariationModeAgent, "prompt", "Prompt", "Help.") - require.NoError(t, err) - - _, err = createLinkedPrompt(options, Selection{ - Project: syncapi.Project{Key: "production"}, - Config: syncapi.Config{ - Key: "support", Mode: syncdomain.VariationModeAgent, - Variations: []syncdomain.Variation{{Key: "prompt"}}, - }, - ModelConfig: syncapi.ModelConfig{Key: "claude", ID: "claude-3-5-sonnet-20241022"}, - Key: "prompt", - Name: "Prompt", - }, prompt) - - require.ErrorContains(t, err, `variation "prompt" already exists`) - content, readErr := os.ReadFile(sourcePath) - require.NoError(t, readErr) - require.Empty(t, content) -} - -func TestCreateDoesNotOverwriteLinkedFileChangedAfterRead(t *testing.T) { - root := t.TempDir() - sourcePath := filepath.Join(root, "prompt.md") - require.NoError(t, os.WriteFile(sourcePath, nil, 0o644)) - options := Options{ - Store: synclocal.NewStore(root), - RepositoryRoot: root, - WorkingDirectory: root, - File: "prompt.md", - Format: syncreference.PlainMarkdown, - } - prompt, err := readLinkedPrompt(options) - require.NoError(t, err) - prompt.content = []byte("content collected by the linker") - - require.NoError(t, os.WriteFile(sourcePath, []byte("newer content from the editor"), 0o644)) - - _, err = createLinkedPrompt(options, Selection{ - Project: syncapi.Project{Key: "production"}, - Config: syncapi.Config{Key: "support", Mode: syncdomain.VariationModeAgent}, - ModelConfig: syncapi.ModelConfig{Key: "claude", ID: "claude-3-5-sonnet-20241022"}, - Key: "prompt", - Name: "Prompt", - }, prompt) - require.ErrorContains(t, err, "changed while syncing") - - content, readErr := os.ReadFile(sourcePath) - require.NoError(t, readErr) - require.Equal(t, "newer content from the editor", string(content)) -} - -func TestPromptForSelectionSkipsTheProjectQuestionAndNamesAMissingProject(t *testing.T) { - catalog := &missingProjectCatalog{} - - _, _, _, err := promptForSelection(Options{ - Catalog: catalog, Input: bytes.NewBuffer(nil), Output: bytes.NewBuffer(nil), ProjectKey: "demo", - }, linkedPrompt{}) - - require.EqualError(t, err, `project "demo" was not found in LaunchDarkly, or the access token cannot read it`) - require.Equal(t, []string{"demo"}, catalog.configSearches) -} - -// missingProjectCatalog fails a project search, and answers a config search -// with a 404 as LaunchDarkly does for a project that does not exist. -type missingProjectCatalog struct { - fakeCatalog - configSearches []string -} - -func (*missingProjectCatalog) SearchProjects(string, int, int) (syncapi.Page[syncapi.Project], error) { - return syncapi.Page[syncapi.Project]{}, errors.New("asked for the project") -} - -func (catalog *missingProjectCatalog) SearchConfigs( - projectKey, _ string, _ []syncdomain.VariationMode, _, _ int, -) (syncapi.Page[syncapi.Config], error) { - catalog.configSearches = append(catalog.configSearches, projectKey) - return syncapi.Page[syncapi.Config]{}, errors.New(`{"code":"not_found","statusCode":404}`) -} - -type fakeCatalog struct { - config syncapi.Config - model syncapi.ModelConfig -} - -func (catalog *fakeCatalog) Config(string, string) (syncapi.Config, error) { - return catalog.config, nil -} - -func (*fakeCatalog) SearchProjects(string, int, int) (syncapi.Page[syncapi.Project], error) { - return syncapi.Page[syncapi.Project]{}, nil -} - -func (*fakeCatalog) SearchConfigs(string, string, []syncdomain.VariationMode, int, int) (syncapi.Page[syncapi.Config], error) { - return syncapi.Page[syncapi.Config]{}, nil -} - -func (catalog *fakeCatalog) ModelConfigs(string) ([]syncapi.ModelConfig, error) { - return []syncapi.ModelConfig{catalog.model}, nil -} - -func create(options Options, selection Selection) (string, error) { - prompt, err := readLinkedPrompt(options) - if err != nil { - return "", err - } - return createLinkedPrompt(options, selection, prompt) -} diff --git a/internal/sync/local/compile.go b/internal/sync/local/compile.go index 1c16f6fa6..6c92875d1 100644 --- a/internal/sync/local/compile.go +++ b/internal/sync/local/compile.go @@ -28,9 +28,7 @@ func (e ParseError) Unwrap() error { return e.Err } // Compile reads every variation in a file system that contains a managed // directory. Tests use it with an in-memory file system. func Compile(fsys fs.FS) ([]syncdomain.SyncedResource, error) { - return compile(fsys, func(reference Reference) ([]byte, error) { - return readReferenceFromFS(fsys, reference) - }) + return compile(fsys) } // CompileWorkspace reads every variation in a repository. It rejects a @@ -40,9 +38,7 @@ func CompileWorkspace(repositoryRoot string) ([]syncdomain.SyncedResource, error if err != nil { return nil, fmt.Errorf("resolve repository root: %w", err) } - return compile(workspaceFS{root: root}, func(reference Reference) ([]byte, error) { - return ReadReference(root, reference) - }) + return compile(workspaceFS{root: root}) } // Compile reads every variation in the repository of the store. @@ -72,7 +68,7 @@ func (fsys workspaceFS) Open(name string) (fs.File, error) { } // compile reads the variation files of each project, in identity order. -func compile(fsys fs.FS, readReference func(Reference) ([]byte, error)) ([]syncdomain.SyncedResource, error) { +func compile(fsys fs.FS) ([]syncdomain.SyncedResource, error) { entries, err := fs.ReadDir(fsys, syncdomain.RootDir) if errors.Is(err, fs.ErrNotExist) { return nil, ErrNoDirectory @@ -88,7 +84,7 @@ func compile(fsys fs.FS, readReference func(Reference) ([]byte, error)) ([]syncd if !entry.IsDir() { continue } - projectVariations, err := compileProject(fsys, entry.Name(), readReference) + projectVariations, err := compileProject(fsys, entry.Name()) if err != nil { return nil, err } @@ -105,7 +101,6 @@ func compile(fsys fs.FS, readReference func(Reference) ([]byte, error)) ([]syncd func compileProject( fsys fs.FS, projectKey string, - readReference func(Reference) ([]byte, error), ) ([]syncdomain.SyncedResource, error) { configsRoot := path.Join(syncdomain.RootDir, projectKey, configsDir) readAttachment := func(kind syncdomain.AttachmentKind, key string) (syncdomain.Attachment, error) { @@ -135,7 +130,6 @@ func compileProject( ProjectKey: projectKey, RelPath: strings.TrimPrefix(name, configsRoot+"/"), Data: data, - ReadReference: readReference, ReadAttachment: readAttachment, }) if err != nil { diff --git a/internal/sync/local/reference.go b/internal/sync/local/reference.go deleted file mode 100644 index 1ca8adeea..000000000 --- a/internal/sync/local/reference.go +++ /dev/null @@ -1,120 +0,0 @@ -package local - -import ( - "errors" - "fmt" - "io/fs" - "os" - "path" - "path/filepath" - "strings" - - syncdomain "github.com/launchdarkly/ldcli/internal/sync" -) - -// Reference is the "ref" field of a linked variation. It names a file in the -// repository and the format of that file. -type Reference = syncdomain.Reference - -// NewReference converts a path that the user entered to a reference. The -// path can be relative to workingDirectory. The file must be in the repository. -func NewReference(repositoryRoot, workingDirectory, file, format string) (Reference, error) { - if file == "" { - return Reference{}, errors.New("linked file path is required") - } - if !filepath.IsAbs(file) { - file = filepath.Join(workingDirectory, file) - } - // Compare resolved paths. A path that looks like it is in the repository - // can go out of it through a symbolic link. - target, err := filepath.EvalSymlinks(file) - if err != nil { - return Reference{}, fmt.Errorf("resolve linked file %q: %w", file, err) - } - root, err := filepath.EvalSymlinks(repositoryRoot) - if err != nil { - return Reference{}, fmt.Errorf("resolve repository root: %w", err) - } - if !isWithin(root, target) { - return Reference{}, errors.New("linked file must be inside the Git repository") - } - - relative, _ := filepath.Rel(root, target) - reference := Reference{File: filepath.ToSlash(relative), Format: format} - if _, err := resolveReferencePath(root, reference); err != nil { - return Reference{}, err - } - return reference, nil -} - -// validateReference makes sure that the file is a clean path in the -// repository and outside the managed directory. -func validateReference(reference Reference) error { - switch { - case reference.File == "": - return errors.New("ref.file is required") - case reference.Format == "": - return errors.New("ref.format is required") - case !fs.ValidPath(reference.File): - return fmt.Errorf("ref.file %q must be a repository-relative path", reference.File) - case reference.File == syncdomain.RootDir || strings.HasPrefix(reference.File, syncdomain.RootDir+"/"): - return fmt.Errorf("ref.file %q must be outside %s", reference.File, syncdomain.RootDir) - case path.Clean(reference.File) != reference.File: - return fmt.Errorf("ref.file %q must be a clean repository-relative path", reference.File) - default: - return nil - } -} - -// resolveReferencePath returns the absolute path of a linked file. It -// resolves symbolic links on each call, because a link can change after the -// variation was linked. -func resolveReferencePath(repositoryRoot string, reference Reference) (string, error) { - if err := validateReference(reference); err != nil { - return "", err - } - root, err := filepath.EvalSymlinks(repositoryRoot) - if err != nil { - return "", fmt.Errorf("resolve repository root: %w", err) - } - target, err := filepath.EvalSymlinks(filepath.Join(root, filepath.FromSlash(reference.File))) - if err != nil { - return "", fmt.Errorf("resolve referenced file %q: %w", reference.File, err) - } - if !isWithin(root, target) { - return "", fmt.Errorf("referenced file %q resolves outside the Git repository", reference.File) - } - info, err := os.Stat(target) - if err != nil { - return "", fmt.Errorf("inspect referenced file %q: %w", reference.File, err) - } - if !info.Mode().IsRegular() { - return "", fmt.Errorf("referenced file %q is not a regular file", reference.File) - } - return target, nil -} - -func readReferenceFromFS(fsys fs.FS, reference Reference) ([]byte, error) { - if err := validateReference(reference); err != nil { - return nil, err - } - data, err := fs.ReadFile(fsys, reference.File) - if err != nil { - return nil, fmt.Errorf("read referenced file %q: %w", reference.File, err) - } - return data, nil -} - -// ReadReference reads a linked file. It resolves symbolic links and makes sure -// that the file is a regular file in the repository. -func ReadReference(repositoryRoot string, reference Reference) ([]byte, error) { - target, err := resolveReferencePath(repositoryRoot, reference) - if err != nil { - return nil, err - } - data, err := os.ReadFile(target) - if err != nil { - return nil, fmt.Errorf("read referenced file %q: %w", reference.File, err) - } - return data, nil -} diff --git a/internal/sync/local/reference_test.go b/internal/sync/local/reference_test.go deleted file mode 100644 index a1cbc3fc7..000000000 --- a/internal/sync/local/reference_test.go +++ /dev/null @@ -1,185 +0,0 @@ -package local - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" - - syncdomain "github.com/launchdarkly/ldcli/internal/sync" - syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" -) - -func TestNewReferenceStoresRepositoryRelativePath(t *testing.T) { - root := t.TempDir() - workingDirectory := filepath.Join(root, "app") - require.NoError(t, os.MkdirAll(filepath.Join(workingDirectory, "prompts"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(workingDirectory, "prompts", "support.md"), []byte("Help"), 0o644)) - - reference, err := NewReference(root, workingDirectory, "prompts/support.md", syncreference.PlainMarkdown) - - require.NoError(t, err) - require.Equal(t, Reference{File: "app/prompts/support.md", Format: syncreference.PlainMarkdown}, reference) -} - -func TestNewReferenceRejectsUnsafePaths(t *testing.T) { - root := t.TempDir() - outside := filepath.Join(t.TempDir(), "prompt.md") - require.NoError(t, os.WriteFile(outside, []byte("Help"), 0o644)) - - _, err := NewReference(root, root, outside, syncreference.PlainMarkdown) - require.ErrorContains(t, err, "inside the Git repository") - - managed := filepath.Join(root, syncdomain.RootDir, "prompt.md") - require.NoError(t, os.MkdirAll(filepath.Dir(managed), 0o755)) - require.NoError(t, os.WriteFile(managed, []byte("Help"), 0o644)) - _, err = NewReference(root, root, managed, syncreference.PlainMarkdown) - require.ErrorContains(t, err, "must be outside") -} - -func TestNewReferenceRejectsSymlinkEscape(t *testing.T) { - root := t.TempDir() - outside := filepath.Join(t.TempDir(), "prompt.md") - require.NoError(t, os.WriteFile(outside, []byte("Help"), 0o644)) - link := filepath.Join(root, "prompt.md") - require.NoError(t, os.Symlink(outside, link)) - - _, err := NewReference(root, root, link, syncreference.PlainMarkdown) - - require.ErrorContains(t, err, "inside the Git repository") -} - -func TestCompileWorkspaceReadsLinkedPrompt(t *testing.T) { - root := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(root, "prompts"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(root, "prompts", "support.md"), []byte("Be helpful.\n"), 0o644)) - _, err := NewStore(root).Add([]VariationFile{{ - ProjectKey: "project", - ConfigKey: "support", - Upsert: true, - Ref: &Reference{File: "prompts/support.md", Format: syncreference.PlainMarkdown}, - Variation: syncdomain.Variation{ - Mode: syncdomain.VariationModeAgent, - Key: "default", - Name: "Default", - }, - }}) - require.NoError(t, err) - - resources, err := CompileWorkspace(root) - - require.NoError(t, err) - require.Len(t, resources, 1) - variation := resources[0].Variation - require.Equal(t, "Be helpful.", variation.Instructions) -} - -func TestReplaceVariationsUpdatesLinkedWrapperAndPrompt(t *testing.T) { - root := t.TempDir() - referencePath := filepath.Join(root, "prompts", "support.md") - require.NoError(t, os.MkdirAll(filepath.Dir(referencePath), 0o755)) - require.NoError(t, os.WriteFile(referencePath, []byte("Old prompt\n"), 0o640)) - store := NewStore(root) - reference := &Reference{File: "prompts/support.md", Format: syncreference.PlainMarkdown} - _, err := store.Add([]VariationFile{{ - ProjectKey: "project", ConfigKey: "support", Upsert: true, Ref: reference, - Variation: syncdomain.Variation{ - Mode: syncdomain.VariationModeAgent, Key: "default", Name: "Old", Instructions: "Old prompt", - }, - }}) - require.NoError(t, err) - - serverVariation := syncdomain.Variation{ - Mode: syncdomain.VariationModeAgent, Key: "default", Name: "New", - Instructions: "First line.\r\nSecond line.\rThird line.", - } - paths, err := store.ReplaceVariations([]VariationReplacement{{ - ProjectKey: "project", - ConfigKey: "support", - Variation: serverVariation, - }}) - - require.NoError(t, err) - require.Equal(t, []string{"project/configs/support/default.prompt.md"}, paths) - content, err := os.ReadFile(referencePath) - require.NoError(t, err) - require.Equal(t, "First line.\nSecond line.\nThird line.\n", string(content)) - info, err := os.Stat(referencePath) - require.NoError(t, err) - require.Equal(t, os.FileMode(0o640), info.Mode().Perm()) - wrapper, err := os.ReadFile(filepath.Join(root, syncdomain.RootDir, paths[0])) - require.NoError(t, err) - require.Contains(t, string(wrapper), "name: New") - require.Contains(t, string(wrapper), "file: prompts/support.md") - - compiled, err := CompileWorkspace(root) - require.NoError(t, err) - resource := requireVariationResource(t, compiled, "support/default") - localVariation := resource.Variation - serverFingerprint, err := syncdomain.FingerprintVariation("project", "support/default", serverVariation) - require.NoError(t, err) - localFingerprint, err := syncdomain.FingerprintVariation("project", "support/default", localVariation) - require.NoError(t, err) - require.Equal(t, serverFingerprint, localFingerprint) -} - -func TestReplaceVariationsDoesNotChangeLinkedFilesWhenServerPromptIsNotRepresentable(t *testing.T) { - root := t.TempDir() - referencePath := filepath.Join(root, "prompt.md") - require.NoError(t, os.WriteFile(referencePath, []byte("Old prompt\n"), 0o644)) - store := NewStore(root) - _, err := store.Add([]VariationFile{{ - ProjectKey: "project", ConfigKey: "support", - Ref: &Reference{File: "prompt.md", Format: syncreference.PlainMarkdown}, - Variation: syncdomain.Variation{ - Mode: syncdomain.VariationModeCompletion, Key: "default", Name: "Old", - Messages: []syncdomain.Message{{Role: "system", Content: "Old prompt"}}, - }, - }}) - require.NoError(t, err) - wrapperPath := filepath.Join(root, syncdomain.RootDir, "project", configsDir, "support", "default"+variationFileSuffix) - originalWrapper, err := os.ReadFile(wrapperPath) - require.NoError(t, err) - - _, err = store.ReplaceVariations([]VariationReplacement{{ - ProjectKey: "project", ConfigKey: "support", - Variation: syncdomain.Variation{ - Mode: syncdomain.VariationModeCompletion, Key: "default", Name: "New", - Messages: []syncdomain.Message{ - {Role: "system", Content: "System"}, - {Role: "user", Content: "User"}, - }, - }, - }}) - - require.ErrorContains(t, err, "at most one system message") - content, readErr := os.ReadFile(referencePath) - require.NoError(t, readErr) - require.Equal(t, "Old prompt\n", string(content)) - wrapper, readErr := os.ReadFile(wrapperPath) - require.NoError(t, readErr) - require.Equal(t, originalWrapper, wrapper) -} - -func TestSourceFilesIncludesWrappersAndReferences(t *testing.T) { - root := t.TempDir() - require.NoError(t, os.Mkdir(filepath.Join(root, "prompts"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(root, "prompts", "support.md"), []byte("Help"), 0o644)) - _, err := NewStore(root).Add([]VariationFile{{ - ProjectKey: "project", ConfigKey: "support", - Ref: &Reference{File: "prompts/support.md", Format: syncreference.PlainMarkdown}, - Variation: syncdomain.Variation{ - Mode: syncdomain.VariationModeAgent, Key: "default", Name: "Default", - }, - }}) - require.NoError(t, err) - - files, err := SourceFiles(root) - - require.NoError(t, err) - require.Equal(t, []string{ - ".launchdarkly/project/configs/support/default.prompt.md", - "prompts/support.md", - }, files) -} diff --git a/internal/sync/local/replace.go b/internal/sync/local/replace.go index 1556d1b8e..6d3146c14 100644 --- a/internal/sync/local/replace.go +++ b/internal/sync/local/replace.go @@ -8,7 +8,6 @@ import ( "path/filepath" syncdomain "github.com/launchdarkly/ldcli/internal/sync" - syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" ) // stagedFile is one file in a replace transaction. The original content lets @@ -25,9 +24,8 @@ type stagedFile struct { } // ReplaceVariations replaces a batch of variation files in one transaction. -// The transaction also writes each linked file and each tool and skill file -// that the variations use. ReplaceVariations returns the replaced variation -// paths. +// The transaction also writes each tool and skill file that the variations +// use. ReplaceVariations returns the replaced variation paths. // // The transaction has three steps. First, it writes every new file to a // temporary path. Then it makes sure that no original changed during the @@ -138,12 +136,11 @@ func (store Store) prepareReplacements(replacements []VariationReplacement) ([]s if err != nil { return nil, err } - // Upsert and ref describe the local file, not LaunchDarkly. Keep them. + // Upsert describes the local file, not LaunchDarkly. Keep it. content, err := renderVariationFile(VariationFile{ ProjectKey: replacement.ProjectKey, ConfigKey: replacement.ConfigKey, Upsert: existing.frontMatter.Upsert, - Ref: existing.frontMatter.Ref, Variation: replacement.Variation, }) if err != nil { @@ -157,15 +154,6 @@ func (store Store) prepareReplacements(replacements []VariationReplacement) ([]s return nil, err } - if ref := existing.frontMatter.Ref; ref != nil { - linked, err := store.prepareLinkedFile(*ref, replacement) - if err != nil { - return nil, err - } - if err := add(linked); err != nil { - return nil, err - } - } attachmentSources = append(attachmentSources, VariationFile{ ProjectKey: replacement.ProjectKey, Variation: replacement.Variation, }) @@ -188,8 +176,7 @@ func (store Store) prepareReplacements(replacements []VariationReplacement) ([]s } // missingVariation describes a variation file that does not exist yet. A new -// file has upsert, so that a later sync can create the variation again, and -// it keeps the link of the variation that it restores. +// file has upsert, so that a later sync can create the variation again. func missingVariation(replacement VariationReplacement) (existingVariation, error) { relativePath, err := variationPath(replacement.ProjectKey, replacement.ConfigKey, replacement.Variation.Key) if err != nil { @@ -198,32 +185,7 @@ func missingVariation(replacement VariationReplacement) (existingVariation, erro return existingVariation{ relativePath: relativePath, mode: 0o644, - frontMatter: variationFrontMatter{Upsert: true, Ref: replacement.Ref}, - }, nil -} - -// prepareLinkedFile renders the new content of the external file that a -// linked variation uses. -func (store Store) prepareLinkedFile(ref Reference, replacement VariationReplacement) (stagedFile, error) { - path, err := resolveReferencePath(store.repositoryRoot, ref) - if err != nil { - return stagedFile{}, err - } - content, err := syncreference.Render(ref.Format, replacement.Variation) - if err != nil { - return stagedFile{}, err - } - info, err := os.Stat(path) - if err != nil { - return stagedFile{}, fmt.Errorf("inspect referenced file %q: %w", ref.File, err) - } - original, err := os.ReadFile(path) - if err != nil { - return stagedFile{}, fmt.Errorf("read referenced file %q: %w", ref.File, err) - } - return stagedFile{ - displayPath: ref.File, path: path, original: original, originalExists: true, - replacement: content, mode: info.Mode().Perm(), + frontMatter: variationFrontMatter{Upsert: true}, }, nil } @@ -299,8 +261,12 @@ func verifyOriginals(files []stagedFile) error { // commitFiles renames each staged file into place. Each rename is atomic, but // the batch is not, so a failure restores the files that commitFiles replaced. func commitFiles(files []stagedFile) error { + return commitFilesWithRename(files, os.Rename) +} + +func commitFilesWithRename(files []stagedFile, rename func(string, string) error) error { for index, file := range files { - if err := os.Rename(file.stagedPath, file.path); err != nil { + if err := rename(file.stagedPath, file.path); err != nil { return errors.Join(fmt.Errorf("replace variation %s: %w", file.displayPath, err), restoreFiles(files[:index])) } } diff --git a/internal/sync/local/sources.go b/internal/sync/local/sources.go index ac58271b0..4bf3716e6 100644 --- a/internal/sync/local/sources.go +++ b/internal/sync/local/sources.go @@ -6,15 +6,14 @@ import ( "os" "path/filepath" "slices" - "strings" syncdomain "github.com/launchdarkly/ldcli/internal/sync" ) // SourceFiles returns every file that can change the sync plan, relative to // the repository root. The list has each file in a project directory and -// each file that a variation links to. A file that is not valid stays in the -// list, so that a fix to the file starts a new sync in watch mode. +// each managed file. A file that is not valid stays in the list, so that a +// fix to the file starts a new sync in watch mode. func SourceFiles(repositoryRoot string) ([]string, error) { managedRoot := filepath.Join(repositoryRoot, syncdomain.RootDir) var files []string @@ -36,11 +35,6 @@ func SourceFiles(repositoryRoot string) ([]string, error) { if filepath.Dir(filePath) != managedRoot { files = append(files, filepath.ToSlash(relative)) } - if strings.HasSuffix(entry.Name(), variationFileSuffix) { - if reference, ok := linkedFile(filePath); ok { - files = append(files, reference) - } - } return nil }) if errors.Is(err, os.ErrNotExist) { @@ -53,20 +47,3 @@ func SourceFiles(repositoryRoot string) ([]string, error) { slices.Sort(files) return slices.Compact(files), nil } - -// linkedFile returns the file that a variation file links to. It ignores a -// variation file that it cannot parse, because compile reports that error. -func linkedFile(file string) (string, bool) { - content, err := os.ReadFile(file) - if err != nil { - return "", false - } - var metadata variationFrontMatter - if _, err := parseYAMLFrontMatter(content, &metadata); err != nil { - return "", false - } - if metadata.Ref == nil || validateReference(*metadata.Ref) != nil { - return "", false - } - return metadata.Ref.File, true -} diff --git a/internal/sync/local/store.go b/internal/sync/local/store.go index 1a63687cc..03a2c54cb 100644 --- a/internal/sync/local/store.go +++ b/internal/sync/local/store.go @@ -22,18 +22,14 @@ type VariationFile struct { ProjectKey string ConfigKey string Upsert bool - Ref *Reference Variation syncdomain.Variation } -// VariationReplacement is new content for a variation file. If -// CreateIfMissing is true and the file does not exist, the replace creates it -// with Ref as its link. An existing file keeps its own link. +// VariationReplacement is new content for a variation file. type VariationReplacement struct { ProjectKey string ConfigKey string CreateIfMissing bool - Ref *Reference Variation syncdomain.Variation } diff --git a/internal/sync/local/variation.go b/internal/sync/local/variation.go index 489b362a2..eff798f1c 100644 --- a/internal/sync/local/variation.go +++ b/internal/sync/local/variation.go @@ -12,7 +12,6 @@ import ( "gopkg.in/yaml.v3" syncdomain "github.com/launchdarkly/ldcli/internal/sync" - syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" ) // A variation file has YAML front matter and a prompt body: @@ -29,29 +28,24 @@ import ( // // // An agent body is the instructions. A completion body is a list of role -// blocks. A completion body without role blocks is one system message. A -// linked variation has a "ref" field and no body. +// blocks. A completion body without role blocks is one system message. const variationFormatVersion = 1 type variationFrontMatter struct { - FormatVersion int `yaml:"formatVersion"` - Upsert bool `yaml:"upsert"` - Ref *Reference `yaml:"ref,omitempty"` + FormatVersion int `yaml:"formatVersion"` + Upsert bool `yaml:"upsert"` syncdomain.Variation `yaml:",inline"` } -// localFile is one variation file and the readers for the files that it references. +// localFile is one variation file and its attachment reader. type localFile struct { ProjectKey string RelPath string Data []byte - ReadReference func(Reference) ([]byte, error) ReadAttachment func(syncdomain.AttachmentKind, string) (syncdomain.Attachment, error) } -// parseVariation reads one variation file, including the content of a linked -// file and every attachment that it references. RelPath is relative to the -// configs directory, for example "support/default.prompt.md". +// parseVariation reads one variation file and every attachment it references. func parseVariation(file localFile) (syncdomain.SyncedResource, error) { meta, body, err := parseVariationFrontMatter(file.RelPath, file.Data) if err != nil { @@ -59,23 +53,7 @@ func parseVariation(file localFile) (syncdomain.SyncedResource, error) { } variation := meta.Variation - if meta.Ref != nil { - // The variation file owns the identity and the model settings. The - // linked file owns only the prompt content. - if strings.TrimSpace(string(body)) != "" { - return syncdomain.SyncedResource{}, errors.New("referenced variation cannot also contain an inline prompt body") - } - content, err := file.ReadReference(*meta.Ref) - if err != nil { - return syncdomain.SyncedResource{}, err - } - if err := syncreference.ApplyToVariation(meta.Ref.Format, content, &variation); err != nil { - return syncdomain.SyncedResource{}, err - } - if stem := variationStem(file.RelPath); variation.Key != stem { - return syncdomain.SyncedResource{}, fmt.Errorf("referenced prompt key %q does not match filename %q", variation.Key, stem) - } - } else if err := ParsePromptBody(string(body), &variation); err != nil { + if err := ParsePromptBody(string(body), &variation); err != nil { return syncdomain.SyncedResource{}, err } @@ -89,7 +67,6 @@ func parseVariation(file localFile) (syncdomain.SyncedResource, error) { ProjectKey: file.ProjectKey, LookupKey: configKey + "/" + variation.Key, Upsert: meta.Upsert, - Ref: meta.Ref, Variation: variation, }, nil } @@ -149,16 +126,9 @@ func renderVariationFile(file VariationFile) ([]byte, error) { case variation.Mode == syncdomain.VariationModeCompletion && variation.Instructions != "": return nil, fmt.Errorf("completion variation %q cannot contain instructions", variation.Key) } - if file.Ref != nil { - if err := validateReference(*file.Ref); err != nil { - return nil, err - } - } - frontMatter, err := marshalYAML(variationFrontMatter{ FormatVersion: variationFormatVersion, Upsert: file.Upsert, - Ref: file.Ref, Variation: variation, }) if err != nil { @@ -170,9 +140,7 @@ func renderVariationFile(file VariationFile) ([]byte, error) { content.Write(frontMatter) content.WriteString("---\n") - // A linked variation has no body, so that only one file holds the prompt. switch { - case file.Ref != nil: case variation.Mode == syncdomain.VariationModeAgent: if instructions := syncdomain.NormalizePromptText(variation.Instructions); instructions != "" { fmt.Fprintf(&content, "\n%s\n", instructions) diff --git a/internal/sync/manifest/baseline_test.go b/internal/sync/manifest/baseline_test.go index 84afcc53c..870752377 100644 --- a/internal/sync/manifest/baseline_test.go +++ b/internal/sync/manifest/baseline_test.go @@ -276,25 +276,3 @@ func (lock *memoryLock) WriteLock(content []byte) error { lock.content = content return nil } - -func TestLockKeepsTheLinkOfAVariation(t *testing.T) { - manifest := Manifest{Resources: []Resource{{ - ResourceKind: syncdomain.KindVariation, ProjectKey: "production", LookupKey: "support/default", - Fingerprint: fingerprint("a"), Version: 1, - }}} - ref := &syncdomain.Reference{File: "prompts/support.md", Format: "plain-markdown"} - manifest.SetRefs([]syncdomain.SyncedResource{{ - Kind: syncdomain.KindVariation, ProjectKey: "production", LookupKey: "support/default", Ref: ref, - }}) - - content, err := encodeLock(manifest) - require.NoError(t, err) - assert.Contains(t, string(content), " ref:\n file: prompts/support.md\n format: plain-markdown\n") - decoded, err := decodeLock(content) - require.NoError(t, err) - assert.Equal(t, ref, decoded.Ref(variationID())) - - // A variation without a local file keeps its recorded link. - manifest.SetRefs(nil) - assert.Equal(t, ref, manifest.Ref(variationID())) -} diff --git a/internal/sync/manifest/model.go b/internal/sync/manifest/model.go index d3c64345d..b7b860ea8 100644 --- a/internal/sync/manifest/model.go +++ b/internal/sync/manifest/model.go @@ -30,9 +30,6 @@ type Resource struct { // uses for optimistic locking. Only the remote manifest has it. The lock // does not store it, so that a lock entry changes only with its content. Version int `yaml:"-"` - // Ref is the linked prompt file of a variation. Only sync.lock stores it, - // so that sync can restore a missing variation file with its link. - Ref *syncdomain.Reference `yaml:"ref,omitempty"` } // ID returns the identity of the resource. @@ -71,24 +68,6 @@ func (manifest *Manifest) Remove(id syncdomain.ResourceID) { } } -// SetRefs records the linked prompt file of each variation. A variation -// without a local file keeps its recorded link. -func (manifest *Manifest) SetRefs(variations []syncdomain.SyncedResource) { - for _, variation := range variations { - if index := manifest.index(variation.ID()); index >= 0 { - manifest.Resources[index].Ref = variation.Ref - } - } -} - -// Ref returns the recorded linked prompt file of a resource, or nil. -func (manifest Manifest) Ref(id syncdomain.ResourceID) *syncdomain.Reference { - if index := manifest.index(id); index >= 0 { - return manifest.Resources[index].Ref - } - return nil -} - // SetAttachments records the state of each tool and skill. Many variations // can share one attachment, but the manifest tracks it once. func (manifest *Manifest) SetAttachments(projectKey string, attachments []syncdomain.Attachment) error { diff --git a/internal/sync/prompt/acceptance_test.go b/internal/sync/prompt/acceptance_test.go index 16affe581..3da41cbe9 100644 --- a/internal/sync/prompt/acceptance_test.go +++ b/internal/sync/prompt/acceptance_test.go @@ -28,7 +28,6 @@ import ( synclocal "github.com/launchdarkly/ldcli/internal/sync/local" syncmanifest "github.com/launchdarkly/ldcli/internal/sync/manifest" syncprompt "github.com/launchdarkly/ldcli/internal/sync/prompt" - syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" ) type directAPI struct { @@ -1211,63 +1210,6 @@ func TestPromptPullPreservesLocalModelOverridesAcrossVersions(t *testing.T) { requireOnlyReads(t, api.requests) } -func TestPromptPushesLinkedFileContent(t *testing.T) { - root := initRepository(t) - local := variation("Linked") - writeLinkedVariation(t, root, local, "Local linked prompt") - api := &directAPI{} - - _, _, err := runPrompt(t, root, api, "--yes") - - require.NoError(t, err) - require.NotNil(t, api.variation) - assert.Equal(t, "Local linked prompt", api.variation.Instructions) - local.Instructions = "Local linked prompt" - assertManifestFingerprint(t, root, local) -} - -func TestPromptPullsServerContentIntoLinkedFile(t *testing.T) { - root := initRepository(t) - baseline := variation("Baseline") - baseline.Instructions = "Old prompt" - server := variation("Server") - server.Instructions = "Server prompt" - writeLinkedVariation(t, root, baseline, "Old prompt") - writeManifest(t, root, baseline) - api := &directAPI{variation: pointer(server)} - - _, _, err := runPrompt(t, root, api, "--yes") - - require.NoError(t, err) - content, err := os.ReadFile(filepath.Join(root, "prompts", "default.md")) - require.NoError(t, err) - assert.Equal(t, "Server prompt\n", string(content)) - assertManifestFingerprint(t, root, server) -} - -func TestPromptServerDeletionLeavesReferencedFile(t *testing.T) { - root := initRepository(t) - baseline := variation("Baseline") - baseline.Instructions = "Keep this file" - writeLinkedVariation(t, root, baseline, "Keep this file") - writeManifest(t, root, baseline) - api := &directAPI{} - - _, _, err := runPrompt(t, root, api, "--yes") - - require.NoError(t, err) - content, err := os.ReadFile(filepath.Join(root, "prompts", "default.md")) - require.NoError(t, err) - assert.Equal(t, "Keep this file\n", string(content)) - resources, err := synclocal.CompileWorkspace(root) - if errors.Is(err, synclocal.ErrNoDirectory) { - resources = nil - err = nil - } - require.NoError(t, err) - assert.Empty(t, resources) -} - // A working copy that is behind another working copy must pull the newer // LaunchDarkly state. It must not write its older file back to LaunchDarkly. func TestPromptStaleWorkingCopyPullsInsteadOfReverting(t *testing.T) { @@ -1299,61 +1241,6 @@ func TestPromptStaleWorkingCopyPullsInsteadOfReverting(t *testing.T) { assert.Equal(t, other.Name, resources[0].Variation.Name) } -func TestPromptRestoresLinkedVariationWithItsLink(t *testing.T) { - root := initRepository(t) - api := syncLinkedVariation(t, root) - deleteVariationFile(t, root) - - _, _, err := runPrompt(t, root, api, "--yes") - - require.NoError(t, err) - resources, err := synclocal.CompileWorkspace(root) - require.NoError(t, err) - require.Len(t, resources, 1) - require.NotNil(t, resources[0].Ref) - assert.Equal(t, "prompts/default.md", resources[0].Ref.File) - - // An edit in the linked file still reaches LaunchDarkly. - require.NoError(t, os.WriteFile(filepath.Join(root, "prompts", "default.md"), []byte("Edited prompt\n"), 0o644)) - _, _, err = runPrompt(t, root, api, "--yes") - require.NoError(t, err) - assert.Equal(t, "Edited prompt", api.variation.Instructions) -} - -func TestPromptStopsRestoreWhenLinkedFileIsNotSafe(t *testing.T) { - tests := map[string]struct { - change func(t *testing.T, linkedFile string) - wantError string - }{ - "linked file is missing": { - change: func(t *testing.T, linkedFile string) { require.NoError(t, os.Remove(linkedFile)) }, - wantError: `its linked file "prompts/default.md" is not available`, - }, - "linked file has local edits": { - change: func(t *testing.T, linkedFile string) { - require.NoError(t, os.WriteFile(linkedFile, []byte("Local edit\n"), 0o644)) - }, - wantError: `its linked file "prompts/default.md" differs from LaunchDarkly`, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - root := initRepository(t) - api := syncLinkedVariation(t, root) - deleteVariationFile(t, root) - test.change(t, filepath.Join(root, "prompts", "default.md")) - - _, _, err := runPrompt(t, root, api, "--yes") - - require.ErrorContains(t, err, test.wantError) - assert.Equal(t, "Linked prompt", api.variation.Instructions) - _, statErr := os.Stat(filepath.Join(root, syncdomain.RootDir, "production", "configs", "support", "default.prompt.md")) - assert.ErrorIs(t, statErr, os.ErrNotExist) - }) - } -} - func TestPromptDetachArchive(t *testing.T) { tests := map[string]struct { arguments []string @@ -1402,25 +1289,6 @@ func TestPromptDetachArchive(t *testing.T) { // syncLinkedVariation writes a linked variation that matches LaunchDarkly and // runs one sync, so that sync.lock records the link. -func syncLinkedVariation(t *testing.T, root string) *directAPI { - t.Helper() - linked := variation("Linked") - writeLinkedVariation(t, root, linked, "Linked prompt") - linked.Instructions = "Linked prompt" - api := &directAPI{variation: pointer(linked)} - _, _, err := runPrompt(t, root, api, "--yes") - require.NoError(t, err) - return api -} - -func deleteVariationFile(t *testing.T, root string) { - t.Helper() - _, err := synclocal.NewStore(root).DeleteVariations([]synclocal.VariationDeletion{{ - ProjectKey: "production", ConfigKey: "support", VariationKey: "default", - }}) - require.NoError(t, err) -} - func TestPromptFilterSyncsOnlyTheNamedVariation(t *testing.T) { root := initRepository(t) baseline := variation("Baseline") @@ -2543,24 +2411,6 @@ func attachLocalTool(t *testing.T, root string, variation syncdomain.Variation, require.NoError(t, os.WriteFile(toolPath, []byte(content), 0o644)) } -func writeLinkedVariation(t *testing.T, root string, value syncdomain.Variation, content string) { - t.Helper() - referencePath := filepath.Join(root, "prompts", value.Key+".md") - require.NoError(t, os.MkdirAll(filepath.Dir(referencePath), 0o755)) - require.NoError(t, os.WriteFile(referencePath, []byte(content+"\n"), 0o644)) - value.Instructions = strings.TrimSpace(content) - _, err := synclocal.NewStore(root).Add([]synclocal.VariationFile{{ - ProjectKey: "production", - ConfigKey: "support", - Upsert: true, - Ref: &synclocal.Reference{ - File: "prompts/" + value.Key + ".md", Format: syncreference.PlainMarkdown, - }, - Variation: value, - }}) - require.NoError(t, err) -} - func writeManifest(t *testing.T, root string, value syncdomain.Variation) { t.Helper() writeManifestResources(t, root, value) diff --git a/internal/sync/prompt/action.go b/internal/sync/prompt/action.go index 9c4199b0f..6278e892f 100644 --- a/internal/sync/prompt/action.go +++ b/internal/sync/prompt/action.go @@ -8,8 +8,6 @@ import ( syncdomain "github.com/launchdarkly/ldcli/internal/sync" synccreate "github.com/launchdarkly/ldcli/internal/sync/create" - synclink "github.com/launchdarkly/ldcli/internal/sync/link" - syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" ) // CommandAction is one sync operation. scope is the scope of the command. @@ -56,15 +54,6 @@ type DetachAction struct { Archive bool } -// LinkAction creates a variation whose prompt is an external file, and then -// syncs. ProjectKey skips the project question. -type LinkAction struct { - File string - Format string - ProjectKey string - Target *synclink.Target -} - func (action SyncAction) validate(scope syncdomain.Scope) error { if action.Watch && action.DryRun { return errors.New("watch does not support --dry-run") @@ -104,25 +93,6 @@ func (action AttachAction) validate(syncdomain.Scope) error { return validateProjectFlag(action.ProjectKey, action.Target) } -func (action LinkAction) validate(syncdomain.Scope) error { - switch { - case action.File == "": - return errors.New("linked file is required") - case action.Format == "": - return errors.New("--format is required") - case action.Target != nil && action.Target.ModelConfigKey == "": - return errors.New("--model-config-key is required with --to") - } - var target *syncdomain.ResourceID - if action.Target != nil { - target = &action.Target.Variation - } - if err := validateProjectFlag(action.ProjectKey, target); err != nil { - return err - } - return syncreference.ValidateFormat(action.Format) -} - // validateSelections rejects positional selectors outside the prompts scope, // and resources that come both from positional selectors and from target // flags. kind is the kind that --key names. diff --git a/internal/sync/prompt/execute.go b/internal/sync/prompt/execute.go index 2b1aee5fc..a52581e2c 100644 --- a/internal/sync/prompt/execute.go +++ b/internal/sync/prompt/execute.go @@ -99,14 +99,6 @@ func executePlan( execute(index) } - if len(failures) == 0 { - variations, err := compileWorkspace(repositoryRoot) - if err != nil { - failures = append(failures, err) - } else { - next.SetRefs(variations) - } - } return outcomes, next, errors.Join(failures...) } @@ -325,7 +317,6 @@ func applyLocalChange(store synclocal.Store, resource PlannedResource) error { ProjectKey: resource.ID.ProjectKey, ConfigKey: configKey, CreateIfMissing: resource.Local == nil, - Ref: resource.restoreRef, Variation: *resource.Server, }}) return err diff --git a/internal/sync/prompt/plan.go b/internal/sync/prompt/plan.go index db9e9aeeb..a91aea21b 100644 --- a/internal/sync/prompt/plan.go +++ b/internal/sync/prompt/plan.go @@ -81,8 +81,6 @@ type PlannedResource struct { // nameSource is where the name of a local skill comes from: a name in its // file, its key, or a flag of the add that creates it. nameSource string - // restoreRef is the link that a restored variation file keeps. - restoreRef *syncdomain.Reference } // The sources of the name of a skill that sync creates. diff --git a/internal/sync/prompt/runner.go b/internal/sync/prompt/runner.go index 4e7437c99..88f7d9866 100644 --- a/internal/sync/prompt/runner.go +++ b/internal/sync/prompt/runner.go @@ -15,10 +15,8 @@ import ( syncdomain "github.com/launchdarkly/ldcli/internal/sync" syncapi "github.com/launchdarkly/ldcli/internal/sync/api" syncbootstrap "github.com/launchdarkly/ldcli/internal/sync/bootstrap" - syncconsole "github.com/launchdarkly/ldcli/internal/sync/console" syncdetach "github.com/launchdarkly/ldcli/internal/sync/detach" syncinteractive "github.com/launchdarkly/ldcli/internal/sync/interactive" - synclink "github.com/launchdarkly/ldcli/internal/sync/link" synclocal "github.com/launchdarkly/ldcli/internal/sync/local" syncmanifest "github.com/launchdarkly/ldcli/internal/sync/manifest" syncsource "github.com/launchdarkly/ldcli/internal/sync/source" @@ -37,7 +35,6 @@ type Runner struct { client resources.Client bootstrap func(syncbootstrap.Options) error detach func(syncdetach.Options) error - link func(synclink.Options) (string, error) watch watchRunner isTerminal func(io.Reader, io.Writer) bool } @@ -48,7 +45,6 @@ func NewRunner(client resources.Client) Runner { client: client, bootstrap: syncbootstrap.Run, detach: syncdetach.Run, - link: synclink.Run, watch: watchWorkspace, isTerminal: syncinteractive.StreamsAreTerminal, } @@ -81,11 +77,6 @@ func (runner Runner) Run(options Options) error { return runner.runAdd(options, workspace, action) case DetachAction: return runner.runDetach(options, workspace, action) - case LinkAction: - linked, err := runner.runLink(options, workspace, action) - if err != nil || !linked { - return err - } case AttachAction: attached, err := runner.runAttach(options, workspace, action) if err != nil || !attached { @@ -179,29 +170,6 @@ func (runner Runner) runDetach(options Options, workspace syncWorkspace, action }) } -// runLink creates the linked variation file. It reports false when the user -// cancels. -func (runner Runner) runLink(options Options, workspace syncWorkspace, action LinkAction) (bool, error) { - path, err := runner.link(synclink.Options{ - Catalog: runner.api(options), - Store: workspace.local, - RepositoryRoot: workspace.root, - WorkingDirectory: options.WorkingDirectory, - File: action.File, - Format: action.Format, - Input: options.Input, - Output: options.Output, - ProjectKey: action.ProjectKey, - Target: action.Target, - NoInput: options.NoInput, - }) - if err != nil || path == "" { - return false, err - } - _ = syncconsole.New(options.Output).Printf("Linked %s/%s.\n", syncdomain.RootDir, path) - return true, nil -} - // runAttach adds the attachment to a local variation. It reports false when // the user cancels. func (runner Runner) runAttach(options Options, workspace syncWorkspace, action AttachAction) (bool, error) { diff --git a/internal/sync/prompt/runner_test.go b/internal/sync/prompt/runner_test.go index f5adec066..4d7d14725 100644 --- a/internal/sync/prompt/runner_test.go +++ b/internal/sync/prompt/runner_test.go @@ -19,9 +19,7 @@ import ( syncapi "github.com/launchdarkly/ldcli/internal/sync/api" syncbootstrap "github.com/launchdarkly/ldcli/internal/sync/bootstrap" syncdetach "github.com/launchdarkly/ldcli/internal/sync/detach" - synclink "github.com/launchdarkly/ldcli/internal/sync/link" synclocal "github.com/launchdarkly/ldcli/internal/sync/local" - syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" syncsource "github.com/launchdarkly/ldcli/internal/sync/source" ) @@ -132,31 +130,6 @@ func TestRunnerWatchDoesNotRunInitialSync(t *testing.T) { assert.True(t, watchCalled) } -func TestRunnerLinksPrompt(t *testing.T) { - root := initGitRepository(t) - runner := NewRunner(noopResourceClient{}) - linkCalled := false - runner.link = func(options synclink.Options) (string, error) { - linkCalled = true - assert.Equal(t, "prompt.md", options.File) - assert.Equal(t, syncreference.PlainMarkdown, options.Format) - return "", nil - } - - err := runner.Run(Options{ - WorkingDirectory: root, - Action: LinkAction{ - File: "prompt.md", Format: syncreference.PlainMarkdown, - }, - Input: os.Stdin, - Output: io.Discard, - ErrorOutput: io.Discard, - }) - - require.NoError(t, err) - assert.True(t, linkCalled) -} - func TestRunnerDetachesWithoutCallingTheAPI(t *testing.T) { root := initGitRepository(t) runner := NewRunner(noopResourceClient{}) @@ -213,9 +186,6 @@ func TestRunnerChecksAttachmentTerminalOnOutput(t *testing.T) { } func TestValidateOptions(t *testing.T) { - require.ErrorContains(t, validateOptions(Options{ - Action: LinkAction{File: "prompt.md"}, - }), "--format is required") require.ErrorContains(t, validateOptions(Options{ Action: SyncAction{Watch: true, DryRun: true}, }), "watch does not support --dry-run") diff --git a/internal/sync/prompt/state.go b/internal/sync/prompt/state.go index 45519b45b..09f1e539c 100644 --- a/internal/sync/prompt/state.go +++ b/internal/sync/prompt/state.go @@ -11,7 +11,6 @@ import ( syncbootstrap "github.com/launchdarkly/ldcli/internal/sync/bootstrap" synclocal "github.com/launchdarkly/ldcli/internal/sync/local" syncmanifest "github.com/launchdarkly/ldcli/internal/sync/manifest" - syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" ) // workspaceState is everything that one plan depends on. The plan compares @@ -195,9 +194,6 @@ func planVariationRows( server[id] = resource } plan := BuildPlan(tracked, local, server) - for index := range plan.Resources { - planRestore(repositoryRoot, baseline, &plan.Resources[index]) - } return plan, localFilesByID, nil } @@ -434,42 +430,6 @@ func references(variation *syncdomain.Variation, id ResourceID) bool { }) } -// planRestore checks a variation whose local file is missing. The restored -// file must keep the link that sync.lock records. When sync cannot restore -// the link safely, the resource becomes an error and sync writes nothing. -func planRestore(repositoryRoot string, baseline syncmanifest.Baseline, resource *PlannedResource) { - if resource.Action != ActionUpdateLocal || resource.Local != nil { - return - } - fail := func(problem, fix string) { - resource.Action = ActionError - resource.Error = "the variation file is missing, and " + problem + ". " + fix - } - ref := baseline.Lock.Ref(resource.ID) - if ref == nil { - return - } - - content, err := synclocal.ReadReference(repositoryRoot, *ref) - if err != nil { - fail(fmt.Sprintf("its linked file %q is not available (%s)", ref.File, err), - "Restore the files from Git, or stop syncing the variation with detach") - return - } - // The restore writes LaunchDarkly's prompt to the linked file. Allow that - // only when the file already has that prompt, so that no local edit is lost. - linked := *resource.Server - if err := syncreference.ApplyToVariation(ref.Format, content, &linked); err == nil { - fingerprint, err := syncdomain.FingerprintVariation(resource.ID.ProjectKey, resource.ID.LookupKey, linked) - if err == nil && fingerprint == resource.ServerFingerprint { - resource.restoreRef = ref - return - } - } - fail(fmt.Sprintf("its linked file %q differs from LaunchDarkly", ref.File), - "Restore the variation file from Git, and then run sync again") -} - // readServerResource reads one variation from LaunchDarkly with the content // of its tools and skills. func readServerResource(client syncapi.Client, attachments *attachmentCache, id syncdomain.ResourceID) (ServerResource, error) { diff --git a/internal/sync/prompt/watch_test.go b/internal/sync/prompt/watch_test.go index 4b60bdbaa..dcda1b84e 100644 --- a/internal/sync/prompt/watch_test.go +++ b/internal/sync/prompt/watch_test.go @@ -16,8 +16,6 @@ import ( "github.com/stretchr/testify/require" syncdomain "github.com/launchdarkly/ldcli/internal/sync" - synclocal "github.com/launchdarkly/ldcli/internal/sync/local" - syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" ) func TestWatchWorkspaceWaitsForDebouncedChange(t *testing.T) { @@ -162,78 +160,6 @@ func TestWatchWorkspaceRefreshesPlanAfterSourceChangesDuringSync(t *testing.T) { require.NoError(t, <-done) } -func TestWatchWorkspaceRefreshesReferencedFilesBeforeRebuildingPlan(t *testing.T) { - root := t.TempDir() - store := synclocal.NewStore(root) - _, err := store.Add([]synclocal.VariationFile{{ - ProjectKey: "project", - ConfigKey: "config", - Variation: syncdomain.Variation{ - Mode: syncdomain.VariationModeAgent, Key: "prompt", Name: "Prompt", Instructions: "Initial", - }, - }}) - require.NoError(t, err) - wrapper := filepath.Join(root, syncdomain.RootDir, "project", "configs", "config", "prompt.prompt.md") - referencePath := filepath.Join(root, "prompts", "prompt.md") - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var attempts atomic.Int32 - attempted := make(chan struct{}, 3) - done := make(chan error, 1) - go func() { - done <- watchWorkspace(ctx, root, 15*time.Millisecond, func(*sourceWatcher) error { - attempt := attempts.Add(1) - if attempt == 1 { - if err := os.MkdirAll(filepath.Dir(referencePath), 0o755); err != nil { - return err - } - if err := os.WriteFile(referencePath, []byte("Referenced"), 0o644); err != nil { - return err - } - content := `--- -formatVersion: 1 -upsert: true -ref: - file: prompts/prompt.md - format: plain-markdown -mode: agent -key: prompt -name: Prompt ---- -` - if err := os.WriteFile(wrapper, []byte(content), 0o644); err != nil { - return err - } - attempted <- struct{}{} - return errRefreshWatchPlan - } - attempted <- struct{}{} - return nil - }, io.Discard) - }() - - time.Sleep(30 * time.Millisecond) - require.NoError(t, os.WriteFile(wrapper, []byte("trigger"), 0o644)) - for attempt := 0; attempt < 2; attempt++ { - select { - case <-attempted: - case <-time.After(time.Second): - t.Fatal("watch did not rebuild the plan") - } - } - - require.NoError(t, os.WriteFile(referencePath, []byte("Changed"), 0o644)) - select { - case <-attempted: - case <-time.After(time.Second): - t.Fatal("watch did not track the newly referenced file") - } - - cancel() - require.NoError(t, <-done) -} - func TestWatchWorkspaceRetriesEditsMadeDuringSuccessfulSync(t *testing.T) { root := t.TempDir() wrapper := filepath.Join(root, syncdomain.RootDir, "project", "configs", "config", "prompt.prompt.md") @@ -331,41 +257,6 @@ func TestWatchedConflictPromptCancellationStopsWatch(t *testing.T) { require.ErrorIs(t, err, context.Canceled) } -func TestWatchWorkspaceTracksMissingReferencedFileParent(t *testing.T) { - root := t.TempDir() - _, err := synclocal.NewStore(root).Add([]synclocal.VariationFile{{ - ProjectKey: "project", ConfigKey: "config", - Ref: &synclocal.Reference{File: "prompts/nested/prompt.md", Format: syncreference.PlainMarkdown}, - Variation: syncdomain.Variation{ - Mode: syncdomain.VariationModeAgent, Key: "prompt", Name: "Prompt", - }, - }}) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - synced := make(chan struct{}, 1) - done := make(chan error, 1) - go func() { - done <- watchWorkspace(ctx, root, 15*time.Millisecond, func(*sourceWatcher) error { - synced <- struct{}{} - return nil - }, io.Discard) - }() - - time.Sleep(30 * time.Millisecond) - referencePath := filepath.Join(root, "prompts", "nested", "prompt.md") - require.NoError(t, os.MkdirAll(filepath.Dir(referencePath), 0o755)) - require.NoError(t, os.WriteFile(referencePath, []byte("created"), 0o644)) - select { - case <-synced: - case <-time.After(time.Second): - t.Fatal("watch did not detect a referenced file created under a missing parent") - } - - cancel() - require.NoError(t, <-done) -} - func TestWatchWorkspaceRewatchesRecreatedManagedTree(t *testing.T) { root := t.TempDir() wrapper := filepath.Join(root, syncdomain.RootDir, "project", "configs", "config", "prompt.prompt.md") @@ -402,43 +293,6 @@ func TestWatchWorkspaceRewatchesRecreatedManagedTree(t *testing.T) { require.NoError(t, <-done) } -func TestWatchWorkspaceDetectsReferencedFileChange(t *testing.T) { - root := t.TempDir() - referencePath := filepath.Join(root, "prompt.md") - require.NoError(t, os.WriteFile(referencePath, []byte("initial"), 0o644)) - _, err := synclocal.NewStore(root).Add([]synclocal.VariationFile{{ - ProjectKey: "project", ConfigKey: "config", - Ref: &synclocal.Reference{File: "prompt.md", Format: syncreference.PlainMarkdown}, - Variation: syncdomain.Variation{ - Mode: syncdomain.VariationModeAgent, Key: "prompt", Name: "Prompt", - }, - }}) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - synced := make(chan struct{}, 1) - done := make(chan error, 1) - var output bytes.Buffer - go func() { - done <- watchWorkspace(ctx, root, 15*time.Millisecond, func(*sourceWatcher) error { - synced <- struct{}{} - return nil - }, &output) - }() - - time.Sleep(30 * time.Millisecond) - require.NoError(t, os.WriteFile(referencePath, []byte("changed"), 0o644)) - select { - case <-synced: - case <-time.After(time.Second): - t.Fatal("watch did not detect referenced file change") - } - cancel() - require.NoError(t, <-done) - require.Contains(t, output.String(), "- .launchdarkly/project/configs/config/prompt.prompt.md\n") - require.Contains(t, output.String(), "- prompt.md\n") -} - func TestSourceWatcherRecognizesNewManagedResourceKinds(t *testing.T) { root := t.TempDir() managedRoot := filepath.Join(root, syncdomain.RootDir) diff --git a/internal/sync/reference/adapters/adapter.go b/internal/sync/reference/adapters/adapter.go deleted file mode 100644 index 4bb1f9317..000000000 --- a/internal/sync/reference/adapters/adapter.go +++ /dev/null @@ -1,20 +0,0 @@ -// Package adapters defines the contract between sync and the external file -// formats that a linked variation can use. -package adapters - -import syncdomain "github.com/launchdarkly/ldcli/internal/sync" - -// Adapter converts between one external file format and a Prompt. -type Adapter interface { - Parse([]byte) (Prompt, error) - Render(Prompt) ([]byte, error) -} - -// Prompt is the content that an external file can supply. An empty field -// means that the file format does not store that value. -type Prompt struct { - Mode syncdomain.VariationMode - Key string - Name string - Messages []syncdomain.Message -} diff --git a/internal/sync/reference/adapters/plainmarkdown/plainmarkdown.go b/internal/sync/reference/adapters/plainmarkdown/plainmarkdown.go deleted file mode 100644 index fa92bac6c..000000000 --- a/internal/sync/reference/adapters/plainmarkdown/plainmarkdown.go +++ /dev/null @@ -1,39 +0,0 @@ -// Package plainmarkdown reads and writes a prompt as one plain Markdown file. -package plainmarkdown - -import ( - "errors" - - syncdomain "github.com/launchdarkly/ldcli/internal/sync" - "github.com/launchdarkly/ldcli/internal/sync/reference/adapters" -) - -// Adapter stores the prompt as one system message. The file has no mode, key, -// or name. -type Adapter struct{} - -var _ adapters.Adapter = Adapter{} - -// Parse reads the file text as one system message. -func (Adapter) Parse(content []byte) (adapters.Prompt, error) { - body := syncdomain.NormalizePromptText(string(content)) - if body == "" { - return adapters.Prompt{}, nil - } - return adapters.Prompt{Messages: []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: body}}}, nil -} - -// Render writes the one system message of the prompt as the file text. -func (Adapter) Render(prompt adapters.Prompt) ([]byte, error) { - if len(prompt.Messages) > 1 || len(prompt.Messages) == 1 && prompt.Messages[0].Role != syncdomain.RoleSystem { - return nil, errors.New("plain-markdown supports at most one system message") - } - if len(prompt.Messages) == 0 { - return nil, nil - } - body := syncdomain.NormalizePromptText(prompt.Messages[0].Content) - if body == "" { - return nil, nil - } - return []byte(body + "\n"), nil -} diff --git a/internal/sync/reference/adapters/plainmarkdown/plainmarkdown_test.go b/internal/sync/reference/adapters/plainmarkdown/plainmarkdown_test.go deleted file mode 100644 index a33149032..000000000 --- a/internal/sync/reference/adapters/plainmarkdown/plainmarkdown_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package plainmarkdown - -import ( - "testing" - - "github.com/stretchr/testify/require" - - syncdomain "github.com/launchdarkly/ldcli/internal/sync" - "github.com/launchdarkly/ldcli/internal/sync/reference/adapters" -) - -func TestAdapterParsesAndRendersRawPrompt(t *testing.T) { - prompt, err := (Adapter{}).Parse([]byte("\nBe helpful.\n")) - - require.NoError(t, err) - require.Equal(t, adapters.Prompt{ - Messages: []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "Be helpful."}}, - }, prompt) - - rendered, err := (Adapter{}).Render(prompt) - require.NoError(t, err) - require.Equal(t, "Be helpful.\n", string(rendered)) -} - -func TestAdapterNormalizesLineEndings(t *testing.T) { - prompt, err := (Adapter{}).Parse([]byte("First line.\r\nSecond line.\rThird line.\r\n")) - - require.NoError(t, err) - require.Equal(t, adapters.Prompt{ - Messages: []syncdomain.Message{{ - Role: syncdomain.RoleSystem, - Content: "First line.\nSecond line.\nThird line.", - }}, - }, prompt) - - rendered, err := (Adapter{}).Render(adapters.Prompt{ - Messages: []syncdomain.Message{{ - Role: syncdomain.RoleSystem, - Content: "First line.\r\nSecond line.\rThird line.", - }}, - }) - - require.NoError(t, err) - require.Equal(t, "First line.\nSecond line.\nThird line.\n", string(rendered)) -} diff --git a/internal/sync/reference/format.go b/internal/sync/reference/format.go deleted file mode 100644 index 0a9b9fc58..000000000 --- a/internal/sync/reference/format.go +++ /dev/null @@ -1,119 +0,0 @@ -// Package reference converts a variation to and from the external file that a -// linked variation uses. Each file format has one adapter. -package reference - -import ( - "fmt" - - syncdomain "github.com/launchdarkly/ldcli/internal/sync" - "github.com/launchdarkly/ldcli/internal/sync/reference/adapters" - "github.com/launchdarkly/ldcli/internal/sync/reference/adapters/plainmarkdown" -) - -// PlainMarkdown identifies the built-in plain Markdown format. -const PlainMarkdown = "plain-markdown" - -// adapterFor is the registry of external formats. To add a format, implement -// adapters.Adapter and add one case here. -func adapterFor(format string) (adapters.Adapter, error) { - switch format { - case PlainMarkdown: - return plainmarkdown.Adapter{}, nil - default: - return nil, fmt.Errorf("unsupported referenced prompt format %q", format) - } -} - -// ValidateFormat reports whether a format has an adapter. -func ValidateFormat(format string) error { - _, err := adapterFor(format) - return err -} - -// Parse reads the content of an external file. -func Parse(format string, content []byte) (adapters.Prompt, error) { - adapter, err := adapterFor(format) - if err != nil { - return adapters.Prompt{}, err - } - return adapter.Parse(content) -} - -// ApplyToVariation replaces the prompt content of the variation with the -// content of an external file. If the file stores a mode, key, or name, that -// value also replaces the value in the variation. -func ApplyToVariation(format string, content []byte, variation *syncdomain.Variation) error { - prompt, err := Parse(format, content) - if err != nil { - return err - } - if prompt.Mode != "" { - if !prompt.Mode.Valid() { - return fmt.Errorf("unsupported referenced prompt mode %q", prompt.Mode) - } - variation.Mode = prompt.Mode - } - if prompt.Key != "" { - variation.Key = prompt.Key - } - if prompt.Name != "" { - variation.Name = prompt.Name - } - for _, message := range prompt.Messages { - if !syncdomain.ValidMessageRole(message.Role) { - return fmt.Errorf("unsupported referenced prompt role %q", message.Role) - } - } - - switch variation.Mode { - case syncdomain.VariationModeAgent: - if !atMostOneSystemMessage(prompt.Messages) { - return fmt.Errorf("agent variation %q requires one system message from its reference", variation.Key) - } - variation.Instructions = "" - variation.Messages = nil - if len(prompt.Messages) == 1 { - variation.Instructions = prompt.Messages[0].Content - } - case syncdomain.VariationModeCompletion: - variation.Instructions = "" - variation.Messages = append([]syncdomain.Message{}, prompt.Messages...) - default: - return fmt.Errorf("referenced prompt does not specify a supported mode") - } - return nil -} - -// Render converts the variation to the content of an external file. -func Render(format string, variation syncdomain.Variation) ([]byte, error) { - adapter, err := adapterFor(format) - if err != nil { - return nil, err - } - - prompt := adapters.Prompt{Mode: variation.Mode, Key: variation.Key, Name: variation.Name} - switch variation.Mode { - case syncdomain.VariationModeAgent: - if len(variation.Messages) != 0 { - return nil, fmt.Errorf("agent variation %q cannot be represented because it contains messages", variation.Key) - } - if variation.Instructions != "" { - prompt.Messages = []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: variation.Instructions}} - } - case syncdomain.VariationModeCompletion: - for _, message := range variation.Messages { - if !syncdomain.ValidMessageRole(message.Role) { - return nil, fmt.Errorf("variation %q has unsupported message role %q", variation.Key, message.Role) - } - } - prompt.Messages = variation.Messages - default: - return nil, fmt.Errorf("referenced prompt does not support variation mode %q", variation.Mode) - } - return adapter.Render(prompt) -} - -// atMostOneSystemMessage reports whether messages is empty or is one system message. -func atMostOneSystemMessage(messages []syncdomain.Message) bool { - return len(messages) == 0 || len(messages) == 1 && messages[0].Role == syncdomain.RoleSystem -} diff --git a/internal/sync/reference/format_test.go b/internal/sync/reference/format_test.go deleted file mode 100644 index b6c20620b..000000000 --- a/internal/sync/reference/format_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package reference - -import ( - "testing" - - "github.com/stretchr/testify/require" - - syncdomain "github.com/launchdarkly/ldcli/internal/sync" -) - -func TestPlainMarkdownAdapterReturnsCommonPromptDomain(t *testing.T) { - prompt, err := Parse(PlainMarkdown, []byte("Be helpful.")) - - require.NoError(t, err) - require.Empty(t, prompt.Mode) - require.Empty(t, prompt.Key) - require.Empty(t, prompt.Name) - require.Equal(t, []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: "Be helpful."}}, prompt.Messages) -} - -func TestPlainMarkdownRoundTrip(t *testing.T) { - tests := []struct { - name string - variation syncdomain.Variation - assert func(*testing.T, syncdomain.Variation) - }{ - { - name: "agent instructions", - variation: syncdomain.Variation{Mode: syncdomain.VariationModeAgent, Key: "prompt"}, - assert: func(t *testing.T, variation syncdomain.Variation) { - require.Equal(t, "Be helpful.", variation.Instructions) - require.Empty(t, variation.Messages) - }, - }, - { - name: "completion system message", - variation: syncdomain.Variation{Mode: syncdomain.VariationModeCompletion, Key: "prompt"}, - assert: func(t *testing.T, variation syncdomain.Variation) { - require.Empty(t, variation.Instructions) - require.Equal(t, []syncdomain.Message{{Role: "system", Content: "Be helpful."}}, variation.Messages) - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := ApplyToVariation(PlainMarkdown, []byte("\nBe helpful.\n"), &test.variation) - require.NoError(t, err) - test.assert(t, test.variation) - - rendered, err := Render(PlainMarkdown, test.variation) - require.NoError(t, err) - require.Equal(t, "Be helpful.\n", string(rendered)) - }) - } -} - -func TestPlainMarkdownRejectsUnrepresentableCompletion(t *testing.T) { - _, err := Render(PlainMarkdown, syncdomain.Variation{ - Mode: syncdomain.VariationModeCompletion, - Key: "prompt", - Messages: []syncdomain.Message{ - {Role: "system", Content: "System"}, - {Role: "user", Content: "User"}, - }, - }) - - require.ErrorContains(t, err, "at most one system message") -} - -func TestReferenceFormatRejectsUnknownFormat(t *testing.T) { - var variation syncdomain.Variation - err := ValidateFormat("anthropic-prompt") - require.ErrorContains(t, err, "unsupported referenced prompt format") - err = ApplyToVariation("anthropic-prompt", nil, &variation) - require.ErrorContains(t, err, "unsupported referenced prompt format") - _, err = Render("anthropic-prompt", variation) - require.ErrorContains(t, err, "unsupported referenced prompt format") -} diff --git a/internal/sync/resource.go b/internal/sync/resource.go index 0ec01adf0..5dcfb77ea 100644 --- a/internal/sync/resource.go +++ b/internal/sync/resource.go @@ -156,9 +156,7 @@ type SyncedResource struct { ProjectKey string LookupKey string // Upsert lets sync create the variation in LaunchDarkly when it is absent. - Upsert bool - // Ref is the linked prompt file, or nil when the variation file holds the prompt. - Ref *Reference + Upsert bool Variation Variation } From f31a6bb316252e49b9c3fcd2772dc2de50a45726 Mon Sep 17 00:00:00 2001 From: Cliff Tawiah Date: Sun, 1 Nov 2026 20:42:16 -0500 Subject: [PATCH 2/5] feat(sync): add projection and problem core --- internal/sync/api/client_test.go | 16 + internal/sync/api/manifest.go | 65 +- internal/sync/attachment.go | 9 + internal/sync/error.go | 106 +++ internal/sync/manifest/baseline_test.go | 44 +- internal/sync/manifest/lock.go | 2 +- internal/sync/manifest/model.go | 82 +++ internal/sync/manifest/store.go | 43 +- internal/sync/manifest/store_test.go | 34 +- internal/sync/projection.go | 697 ++++++++++++++++++ internal/sync/projection_parameters_test.go | 170 +++++ internal/sync/prompt/acceptance_test.go | 6 +- internal/sync/relationship_projection_test.go | 76 ++ internal/sync/resource.go | 45 +- internal/sync/variation.go | 17 +- 15 files changed, 1361 insertions(+), 51 deletions(-) create mode 100644 internal/sync/error.go create mode 100644 internal/sync/projection.go create mode 100644 internal/sync/projection_parameters_test.go create mode 100644 internal/sync/relationship_projection_test.go diff --git a/internal/sync/api/client_test.go b/internal/sync/api/client_test.go index 16d6e8fdb..48de862da 100644 --- a/internal/sync/api/client_test.go +++ b/internal/sync/api/client_test.go @@ -463,6 +463,7 @@ func TestClientArchiveVariationReturnsTransportError(t *testing.T) { func TestClientGetsSyncManifest(t *testing.T) { transport := &recordingClient{Responses: [][]byte{[]byte(`{ + "formatVersion": 1, "source": "git:example.com/acme/repo", "items": [{ "resourceKind": "variation", @@ -487,6 +488,7 @@ func TestClientGetsSyncManifest(t *testing.T) { func TestClientPatchesSyncManifest(t *testing.T) { transport := &recordingClient{Responses: [][]byte{[]byte(`{ + "formatVersion": 1, "source": "git:example.com/acme/repo", "items": [] }`)}} @@ -508,6 +510,7 @@ func TestClientPatchesSyncManifest(t *testing.T) { assert.Equal(t, http.MethodPatch, request.Method) assert.Equal(t, "application/json", request.ContentType) assert.JSONEq(t, `{ + "formatVersion": 1, "source": "git:example.com/acme/repo", "upserts": [{ "resourceKind": "variation", @@ -519,6 +522,19 @@ func TestClientPatchesSyncManifest(t *testing.T) { }`, string(request.Body)) } +func TestClientRejectsUnknownSyncManifestVersion(t *testing.T) { + transport := &recordingClient{Responses: [][]byte{[]byte(`{ + "formatVersion": 99, + "source": "git:example.com/acme/repo", + "items": [] + }`)}} + client := NewClient(transport, "token", "https://example.com") + + _, err := client.GetSyncManifest("project", "git:example.com/acme/repo") + + require.ErrorContains(t, err, "unsupported formatVersion 99") +} + func TestIsConflictRecognizesAPIStatus(t *testing.T) { assert.True(t, IsConflict(errors.New(`{"code":"conflict","statusCode":409}`))) assert.False(t, IsConflict(errors.New(`{"code":"invalid_request","statusCode":400}`))) diff --git a/internal/sync/api/manifest.go b/internal/sync/api/manifest.go index ad997bc9d..958f2605f 100644 --- a/internal/sync/api/manifest.go +++ b/internal/sync/api/manifest.go @@ -9,27 +9,39 @@ import ( syncdomain "github.com/launchdarkly/ldcli/internal/sync" ) +// SyncManifestFormatVersion is the remote sync manifest schema. +const SyncManifestFormatVersion = 1 + // SyncManifest is the sync baseline that LaunchDarkly stores for one project // and one repository source. type SyncManifest struct { - Source string `json:"source"` - Items []SyncManifestResource `json:"items"` + FormatVersion int `json:"formatVersion"` + Source string `json:"source"` + Items []SyncManifestResource `json:"items"` } // SyncManifestResource is one versioned baseline entry. type SyncManifestResource struct { - ResourceKind syncdomain.Kind `json:"resourceKind"` - ResourceLookupKey string `json:"resourceLookupKey"` - Fingerprint string `json:"fingerprint"` - Version int `json:"version"` + ResourceKind syncdomain.Kind `json:"resourceKind"` + ResourceLookupKey string `json:"resourceLookupKey"` + Fingerprint string `json:"fingerprint"` + Version int `json:"version"` + FingerprintSchema string `json:"fingerprintSchema,omitempty"` + ReferenceDigest string `json:"referenceDigest,omitempty"` + AdapterBehaviorVersion int `json:"adapterBehaviorVersion,omitempty"` + AdapterBehaviorDigest string `json:"adapterBehaviorDigest,omitempty"` } // SyncManifestUpsert creates an entry, or updates the entry at Version. type SyncManifestUpsert struct { - ResourceKind syncdomain.Kind `json:"resourceKind"` - ResourceLookupKey string `json:"resourceLookupKey"` - Fingerprint string `json:"fingerprint"` - Version int `json:"version"` + ResourceKind syncdomain.Kind `json:"resourceKind"` + ResourceLookupKey string `json:"resourceLookupKey"` + Fingerprint string `json:"fingerprint"` + Version int `json:"version"` + FingerprintSchema string `json:"fingerprintSchema,omitempty"` + ReferenceDigest string `json:"referenceDigest,omitempty"` + AdapterBehaviorVersion int `json:"adapterBehaviorVersion,omitempty"` + AdapterBehaviorDigest string `json:"adapterBehaviorDigest,omitempty"` } // SyncManifestDeletion removes the entry at Version. @@ -48,7 +60,14 @@ func (client Client) GetSyncManifest(projectKey, source string) (SyncManifest, e if err != nil { return SyncManifest{}, err } - return decodeJSON[SyncManifest](response, fmt.Sprintf("sync manifest for project %q", projectKey)) + manifest, err := decodeJSON[SyncManifest](response, fmt.Sprintf("sync manifest for project %q", projectKey)) + if err != nil { + return SyncManifest{}, err + } + if err := validateSyncManifest(manifest); err != nil { + return SyncManifest{}, fmt.Errorf("sync manifest for project %q: %w", projectKey, err) + } + return manifest, nil } // PatchSyncManifest applies versioned changes and returns the new baseline. @@ -68,10 +87,16 @@ func (client Client) PatchSyncManifest( request := mutation{ method: http.MethodPatch, action: "update", resource: "sync manifest for project", key: projectKey, body: struct { - Source string `json:"source"` - Upserts []SyncManifestUpsert `json:"upserts"` - Deletions []SyncManifestDeletion `json:"deletions"` - }{Source: source, Upserts: upserts, Deletions: deletions}, + FormatVersion int `json:"formatVersion"` + Source string `json:"source"` + Upserts []SyncManifestUpsert `json:"upserts"` + Deletions []SyncManifestDeletion `json:"deletions"` + }{ + FormatVersion: SyncManifestFormatVersion, + Source: source, + Upserts: upserts, + Deletions: deletions, + }, } response, err := client.mutate(request, manifestPath(projectKey)...) if err != nil { @@ -84,9 +109,19 @@ func (client Client) PatchSyncManifest( // caller must read the manifest again to find its state. return SyncManifest{}, newMutationError("decode updated", request.resource, projectKey, err) } + if err := validateSyncManifest(manifest); err != nil { + return SyncManifest{}, newMutationError("decode updated", request.resource, projectKey, err) + } return manifest, nil } +func validateSyncManifest(manifest SyncManifest) error { + if manifest.FormatVersion != SyncManifestFormatVersion { + return fmt.Errorf("unsupported formatVersion %d", manifest.FormatVersion) + } + return nil +} + func manifestPath(projectKey string) []string { return projectPath(projectKey, "configs/sync/manifests") } diff --git a/internal/sync/attachment.go b/internal/sync/attachment.go index 8062cd15b..5f6d62ff1 100644 --- a/internal/sync/attachment.go +++ b/internal/sync/attachment.go @@ -63,6 +63,9 @@ func (skill Skill) CreateName() string { type Attachment struct { Kind AttachmentKind Version int + // Projection records the fields that a linked definition owns. It lets a + // dependency keep unowned LaunchDarkly fields when sync publishes it. + Projection Projection `json:"-" yaml:"-"` // Upsert lets sync create the tool or the skill in LaunchDarkly when it // is absent. Upsert bool @@ -91,6 +94,11 @@ func (attachment Attachment) ID(projectKey string) ResourceID { type SyncedAttachment struct { ProjectKey string Attachment Attachment + // Source is the descriptor-backed repository source, or nil for a managed file. + Source *Reference + Dependencies []string + IndexedSelections []IndexedSelection + Projection Projection } // ID returns the identity of the tool or the skill. @@ -103,6 +111,7 @@ func (attachment SyncedAttachment) ID() ResourceID { // the same content. func CanonicalAttachment(attachment Attachment) Attachment { attachment.Version = 0 + attachment.Projection = nil attachment.Upsert = false if attachment.Tool != nil { tool := *attachment.Tool diff --git a/internal/sync/error.go b/internal/sync/error.go new file mode 100644 index 000000000..9a1d67c33 --- /dev/null +++ b/internal/sync/error.go @@ -0,0 +1,106 @@ +package sync + +import ( + "errors" + "fmt" +) + +// ErrorCode identifies one stable sync failure. +type ErrorCode string + +const ( + ErrorInvalidPathSyntax ErrorCode = "invalid_path_syntax" + ErrorPathNotFound ErrorCode = "path_not_found" + ErrorPathAmbiguous ErrorCode = "path_ambiguous" + ErrorPathTypeMismatch ErrorCode = "path_type_mismatch" + ErrorFilterTypeMismatch ErrorCode = "filter_value_type_mismatch" + ErrorMapTypeMismatch ErrorCode = "map_value_type_mismatch" + ErrorIndexOutOfRange ErrorCode = "index_out_of_range" + ErrorIndexedSelectionChanged ErrorCode = "indexed_selection_changed" + ErrorUnsupportedFormat ErrorCode = "unsupported_source_format" + ErrorFieldReadOnly ErrorCode = "field_read_only" + ErrorDuplicateKey ErrorCode = "duplicate_key" + ErrorInvalidSource ErrorCode = "invalid_source" + ErrorInvalidReference ErrorCode = "invalid_reference" + ErrorDefinitionConflict ErrorCode = "resource_definition_conflict" + ErrorLinkedSourceMissing ErrorCode = "linked_source_missing" + ErrorAttachmentDefinitionMissing ErrorCode = "attachment_definition_missing" + ErrorSyncSettingUnavailable ErrorCode = "sync_setting_unavailable" + ErrorBindingStale ErrorCode = "binding_stale" + ErrorStateChanged ErrorCode = "state_changed" + ErrorPublicationUnsupported ErrorCode = "publication_not_supported" + ErrorOverlappingEdits ErrorCode = "overlapping_edits" + ErrorResourceDeleted ErrorCode = "resource_deleted_in_launchdarkly" + ErrorPlaceholderAmbiguous ErrorCode = "placeholder_translation_ambiguous" + ErrorImportUnresolved ErrorCode = "import_unresolved" +) + +// Problem describes one actionable sync error. +type Problem struct { + Code ErrorCode `json:"code"` + Message string `json:"message"` + Resource string `json:"resource,omitempty"` + File string `json:"file,omitempty"` + Line int `json:"line,omitempty"` + Column int `json:"column,omitempty"` + Suggestion string `json:"suggestion,omitempty"` + Cause error `json:"-"` +} + +// Error returns the message without embedding the stable code. +func (problem *Problem) Error() string { + if problem == nil { + return "" + } + if problem.Suggestion == "" { + return problem.Message + } + return problem.Message + ". " + problem.Suggestion +} + +// Unwrap returns the lower-level cause. +func (problem *Problem) Unwrap() error { + if problem == nil { + return nil + } + return problem.Cause +} + +// NewProblem creates one sync problem. +func NewProblem(code ErrorCode, message, suggestion string) *Problem { + return &Problem{Code: code, Message: message, Suggestion: suggestion} +} + +// ProblemFrom returns the first sync problem in an error chain. +func ProblemFrom(err error) (*Problem, bool) { + var problem *Problem + ok := errors.As(err, &problem) + return problem, ok +} + +// WithResource adds a resource identity to a problem. +func (problem *Problem) WithResource(id ResourceID) *Problem { + if problem != nil { + problem.Resource = id.Label() + } + return problem +} + +// WithFile adds a source location to a problem. +func (problem *Problem) WithFile(file string, line, column int) *Problem { + if problem != nil { + problem.File, problem.Line, problem.Column = file, line, column + } + return problem +} + +// WrapProblem adds a lower-level cause. +func WrapProblem(problem *Problem, cause error) *Problem { + if problem != nil { + problem.Cause = cause + if problem.Message == "" && cause != nil { + problem.Message = fmt.Sprint(cause) + } + } + return problem +} diff --git a/internal/sync/manifest/baseline_test.go b/internal/sync/manifest/baseline_test.go index 870752377..cad70b956 100644 --- a/internal/sync/manifest/baseline_test.go +++ b/internal/sync/manifest/baseline_test.go @@ -41,7 +41,7 @@ resources: func TestDecodeLockRejectsInvalidContent(t *testing.T) { for name, content := range map[string]string{ "unknown field": "formatVersion: 1\nresources: []\nextra: true\n", - "unknown version": "formatVersion: 2\nresources: []\n", + "unknown version": "formatVersion: 3\nresources: []\n", "bad fingerprint": "formatVersion: 1\nresources:\n - kind: tool\n project: p\n key: k\n fingerprint: x\n", } { _, err := decodeLock([]byte(content)) @@ -49,6 +49,34 @@ func TestDecodeLockRejectsInvalidContent(t *testing.T) { } } +func TestLockFormatOneContainsSourceMetadata(t *testing.T) { + content, err := encodeLock(Manifest{Resources: []Resource{{ + ResourceKind: syncdomain.KindSkill, + ProjectKey: "production", + LookupKey: "refunds", + Fingerprint: fingerprint("a"), + FingerprintSchema: syncdomain.ProjectionFingerprintSchema, + ReferenceDigest: fingerprint("b"), + AdapterBehaviorVersion: syncdomain.AdapterBehaviorVersion, + AdapterBehaviorDigest: syncdomain.AdapterBehaviorDigest, + IndexedSelections: []syncdomain.IndexedSelection{{ + Path: "select#0", CollectionLength: 2, Fingerprint: fingerprint("c"), + }}, + }}}) + + require.NoError(t, err) + assert.Contains(t, string(content), "formatVersion: 1") + assert.Contains(t, string(content), "fingerprintSchema: launchdarkly.config.projection/1") + assert.Contains(t, string(content), "indexedSelections:") + assert.Contains(t, string(content), "collectionLength: 2") + assert.NotContains(t, string(content), "file:") + assert.NotContains(t, string(content), "direction:") + decoded, err := decodeLock(content) + require.NoError(t, err) + require.Len(t, decoded.Resources[0].IndexedSelections, 1) + assert.Equal(t, 2, decoded.Resources[0].IndexedSelections[0].CollectionLength) +} + // Other branches add entries to the shared remote manifest. A working copy // without a sync.lock file must not track them. func TestBaselineStoreWithoutLockFileTracksNothing(t *testing.T) { @@ -97,7 +125,7 @@ func TestBaselineStaleComparesFingerprints(t *testing.T) { "newer version with a different fingerprint": {remote: remoteManifest(fingerprint("b"), 5), stale: true}, "newer version with the same fingerprint": {remote: remoteManifest(fingerprint("a"), 5)}, "same version with a different fingerprint": {remote: remoteManifest(fingerprint("b"), 4), stale: true}, - "remote entry removed": {remote: syncapi.SyncManifest{Source: testSource}}, + "remote entry removed": {remote: syncapi.SyncManifest{FormatVersion: syncapi.SyncManifestFormatVersion, Source: testSource}}, } for name, test := range tests { @@ -117,7 +145,8 @@ func TestBaselineStoreSaveSendsRemoteVersionsAndWritesLockWithoutThem(t *testing client := &manifestClient{ manifests: map[string]syncapi.SyncManifest{"production": remoteManifest(fingerprint("b"), 5)}, patchResponses: []syncapi.SyncManifest{{ - Source: testSource, + FormatVersion: syncapi.SyncManifestFormatVersion, + Source: testSource, Items: []syncapi.SyncManifestResource{ {ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "support/default", Fingerprint: fingerprint("c"), Version: 6}, {ResourceKind: syncdomain.KindTool, ResourceLookupKey: "search", Fingerprint: fingerprint("d"), Version: 1}, @@ -181,8 +210,11 @@ func TestBaselineStoreSaveDoesNotWriteLockWhenRemoteSaveFails(t *testing.T) { func TestBaselineStoreSaveRemovesEmptyLock(t *testing.T) { client := &manifestClient{ - manifests: map[string]syncapi.SyncManifest{"production": remoteManifest(fingerprint("a"), 4)}, - patchResponses: []syncapi.SyncManifest{{Source: testSource}}, + manifests: map[string]syncapi.SyncManifest{"production": remoteManifest(fingerprint("a"), 4)}, + patchResponses: []syncapi.SyncManifest{{ + FormatVersion: syncapi.SyncManifestFormatVersion, + Source: testSource, + }}, } lock := lockWith(t, fingerprint("a"), 4) store := NewBaselineStore(NewStore(client, testSource), lock) @@ -246,7 +278,7 @@ func variationID() syncdomain.ResourceID { } func remoteManifest(fingerprint string, version int) syncapi.SyncManifest { - return syncapi.SyncManifest{Source: testSource, Items: []syncapi.SyncManifestResource{{ + return syncapi.SyncManifest{FormatVersion: syncapi.SyncManifestFormatVersion, Source: testSource, Items: []syncapi.SyncManifestResource{{ ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "support/default", Fingerprint: fingerprint, Version: version, }}} } diff --git a/internal/sync/manifest/lock.go b/internal/sync/manifest/lock.go index ac90a4430..44c0b332f 100644 --- a/internal/sync/manifest/lock.go +++ b/internal/sync/manifest/lock.go @@ -12,7 +12,7 @@ import ( // The sync.lock file stores the baseline of one working copy. Engineers commit // it with the .launchdarkly files, so that the baseline moves with the files. const ( - lockFormatVersion = 1 + lockFormatVersion = FormatVersion lockHeader = "# Written by ldcli sync. Commit this file with the .launchdarkly files.\n" ) diff --git a/internal/sync/manifest/model.go b/internal/sync/manifest/model.go index b7b860ea8..aa1909b8d 100644 --- a/internal/sync/manifest/model.go +++ b/internal/sync/manifest/model.go @@ -14,6 +14,9 @@ import ( var fingerprintPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) +// FormatVersion is the local sync manifest and sync.lock schema. +const FormatVersion = 1 + // Manifest is the baseline of every tracked resource. type Manifest struct { Resources []Resource @@ -26,6 +29,15 @@ type Resource struct { ProjectKey string `yaml:"project"` LookupKey string `yaml:"key"` Fingerprint string `yaml:"fingerprint"` + // FingerprintSchema identifies the meaning of a descriptor-backed fingerprint. + FingerprintSchema string `yaml:"fingerprintSchema,omitempty"` + // ReferenceDigest detects a descriptor mapping change. + ReferenceDigest string `yaml:"referenceDigest,omitempty"` + // AdapterBehaviorVersion and AdapterBehaviorDigest detect adapter changes. + AdapterBehaviorVersion int `yaml:"adapterBehaviorVersion,omitempty"` + AdapterBehaviorDigest string `yaml:"adapterBehaviorDigest,omitempty"` + // IndexedSelections detect a changed item at an indexed source path. + IndexedSelections []syncdomain.IndexedSelection `yaml:"indexedSelections,omitempty"` // Version is the version of the remote manifest entry, which LaunchDarkly // uses for optimistic locking. Only the remote manifest has it. The lock // does not store it, so that a lock entry changes only with its content. @@ -61,6 +73,49 @@ func (manifest *Manifest) SetFingerprint(id syncdomain.ResourceID, fingerprint s }) } +// SetSourceMetadata records generated metadata for a descriptor-backed resource. +func (manifest *Manifest) SetSourceMetadata(id syncdomain.ResourceID, reference *syncdomain.Reference) error { + index := manifest.index(id) + if index < 0 { + return nil + } + resource := &manifest.Resources[index] + if reference == nil { + resource.FingerprintSchema = "" + resource.ReferenceDigest = "" + resource.AdapterBehaviorVersion = 0 + resource.AdapterBehaviorDigest = "" + resource.IndexedSelections = nil + return nil + } + digest, err := syncdomain.ReferenceDigest(*reference) + if err != nil { + return err + } + resource.FingerprintSchema = syncdomain.ProjectionFingerprintSchema + resource.ReferenceDigest = digest + resource.AdapterBehaviorVersion = syncdomain.AdapterBehaviorVersion + resource.AdapterBehaviorDigest = syncdomain.AdapterBehaviorDigest + resource.IndexedSelections = nil + return nil +} + +// SetIndexedSelections records the indexed source items for one resource. +func (manifest *Manifest) SetIndexedSelections( + id syncdomain.ResourceID, + selections []syncdomain.IndexedSelection, +) { + index := manifest.index(id) + if index < 0 { + return + } + selections = slices.Clone(selections) + slices.SortFunc(selections, func(left, right syncdomain.IndexedSelection) int { + return strings.Compare(left.Path, right.Path) + }) + manifest.Resources[index].IndexedSelections = selections +} + // Remove stops tracking one resource. func (manifest *Manifest) Remove(id syncdomain.ResourceID) { if index := manifest.index(id); index >= 0 { @@ -157,9 +212,36 @@ func (manifest Manifest) Validate() error { switch { case !fingerprintPattern.MatchString(resource.Fingerprint): return fmt.Errorf("invalid fingerprint for %s", id) + case resource.FingerprintSchema != "" && resource.FingerprintSchema != syncdomain.ProjectionFingerprintSchema: + return fmt.Errorf("unsupported fingerprint schema %q for %s", resource.FingerprintSchema, id) + case resource.FingerprintSchema != "" && + (resource.ReferenceDigest == "" || resource.AdapterBehaviorVersion == 0 || resource.AdapterBehaviorDigest == ""): + return fmt.Errorf("incomplete source metadata for %s", id) + case resource.FingerprintSchema == "" && + (resource.ReferenceDigest != "" || resource.AdapterBehaviorVersion != 0 || + resource.AdapterBehaviorDigest != "" || len(resource.IndexedSelections) != 0): + return fmt.Errorf("source metadata without a fingerprint schema for %s", id) + case resource.ReferenceDigest != "" && !fingerprintPattern.MatchString(resource.ReferenceDigest): + return fmt.Errorf("invalid reference digest for %s", id) + case resource.AdapterBehaviorDigest != "" && !fingerprintPattern.MatchString(resource.AdapterBehaviorDigest): + return fmt.Errorf("invalid adapter behavior digest for %s", id) + case resource.AdapterBehaviorVersion < 0: + return fmt.Errorf("invalid adapter behavior version for %s", id) case resource.Version < 0: return fmt.Errorf("invalid version for %s", id) } + for index, selection := range resource.IndexedSelections { + switch { + case selection.Path == "": + return fmt.Errorf("indexed selection path is empty for %s", id) + case selection.CollectionLength <= 0: + return fmt.Errorf("invalid indexed collection length for %s", id) + case !fingerprintPattern.MatchString(selection.Fingerprint): + return fmt.Errorf("invalid indexed selection fingerprint for %s", id) + case index > 0 && resource.IndexedSelections[index-1].Path >= selection.Path: + return fmt.Errorf("indexed selections are not in path order for %s", id) + } + } if _, duplicate := seen[id]; duplicate { return fmt.Errorf("duplicate manifest resource %s", id) } diff --git a/internal/sync/manifest/store.go b/internal/sync/manifest/store.go index 66c8d9f66..c2a6cec19 100644 --- a/internal/sync/manifest/store.go +++ b/internal/sync/manifest/store.go @@ -133,6 +133,13 @@ func (store Store) patch( } func (store Store) resourcesFromRemote(projectKey string, remote syncapi.SyncManifest) ([]Resource, error) { + if remote.FormatVersion != syncapi.SyncManifestFormatVersion { + return nil, fmt.Errorf( + "sync manifest for project %q has unsupported formatVersion %d", + projectKey, + remote.FormatVersion, + ) + } if remote.Source != "" && remote.Source != store.source { return nil, fmt.Errorf( "sync manifest for project %q returned source %q instead of %q", projectKey, remote.Source, store.source, @@ -147,11 +154,15 @@ func (store Store) resourcesFromRemote(projectKey string, remote syncapi.SyncMan ) } resources = append(resources, Resource{ - ResourceKind: item.ResourceKind, - ProjectKey: projectKey, - LookupKey: item.ResourceLookupKey, - Fingerprint: item.Fingerprint, - Version: item.Version, + ResourceKind: item.ResourceKind, + ProjectKey: projectKey, + LookupKey: item.ResourceLookupKey, + Fingerprint: item.Fingerprint, + Version: item.Version, + FingerprintSchema: item.FingerprintSchema, + ReferenceDigest: item.ReferenceDigest, + AdapterBehaviorVersion: item.AdapterBehaviorVersion, + AdapterBehaviorDigest: item.AdapterBehaviorDigest, }) } return resources, nil @@ -169,14 +180,18 @@ func changes(before, after []Resource) ([]syncapi.SyncManifestUpsert, []syncapi. for _, resource := range after { current, exists := remaining[resource.ID()] delete(remaining, resource.ID()) - if exists && current.Fingerprint == resource.Fingerprint { + if exists && sameManifestContent(current, resource) { continue } upserts = append(upserts, syncapi.SyncManifestUpsert{ - ResourceKind: resource.ResourceKind, - ResourceLookupKey: resource.LookupKey, - Fingerprint: resource.Fingerprint, - Version: current.Version, + ResourceKind: resource.ResourceKind, + ResourceLookupKey: resource.LookupKey, + Fingerprint: resource.Fingerprint, + Version: current.Version, + FingerprintSchema: resource.FingerprintSchema, + ReferenceDigest: resource.ReferenceDigest, + AdapterBehaviorVersion: resource.AdapterBehaviorVersion, + AdapterBehaviorDigest: resource.AdapterBehaviorDigest, }) } @@ -193,6 +208,14 @@ func changes(before, after []Resource) ([]syncapi.SyncManifestUpsert, []syncapi. return upserts, deletions } +func sameManifestContent(left, right Resource) bool { + return left.Fingerprint == right.Fingerprint && + left.FingerprintSchema == right.FingerprintSchema && + left.ReferenceDigest == right.ReferenceDigest && + left.AdapterBehaviorVersion == right.AdapterBehaviorVersion && + left.AdapterBehaviorDigest == right.AdapterBehaviorDigest +} + // changesApplied reports whether the remote manifest contains every change. func changesApplied( remote syncapi.SyncManifest, diff --git a/internal/sync/manifest/store_test.go b/internal/sync/manifest/store_test.go index bc19553a7..a19af8cfe 100644 --- a/internal/sync/manifest/store_test.go +++ b/internal/sync/manifest/store_test.go @@ -52,14 +52,16 @@ func (client *manifestClient) PatchSyncManifest( func TestStoreLoadsProjectManifests(t *testing.T) { client := &manifestClient{manifests: map[string]syncapi.SyncManifest{ "zeta": { - Source: "git:example/repo", + FormatVersion: syncapi.SyncManifestFormatVersion, + Source: "git:example/repo", Items: []syncapi.SyncManifestResource{{ ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "config/b", Fingerprint: fingerprint("b"), Version: 4, }}, }, "alpha": { - Source: "git:example/repo", + FormatVersion: syncapi.SyncManifestFormatVersion, + Source: "git:example/repo", Items: []syncapi.SyncManifestResource{{ ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "config/a", Fingerprint: fingerprint("a"), Version: 2, @@ -78,9 +80,15 @@ func TestStoreLoadsProjectManifests(t *testing.T) { func TestStoreUpdatesWithRemoteVersions(t *testing.T) { client := &manifestClient{patchResponses: []syncapi.SyncManifest{{ - Source: "git:example/repo", + FormatVersion: syncapi.SyncManifestFormatVersion, + Source: "git:example/repo", Items: []syncapi.SyncManifestResource{ - {ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "config/existing", Fingerprint: fingerprint("c"), Version: 4}, + { + ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "config/existing", + Fingerprint: fingerprint("c"), Version: 4, + FingerprintSchema: syncdomain.ProjectionFingerprintSchema, ReferenceDigest: fingerprint("e"), + AdapterBehaviorVersion: syncdomain.AdapterBehaviorVersion, AdapterBehaviorDigest: syncdomain.AdapterBehaviorDigest, + }, {ResourceKind: syncdomain.KindTool, ResourceLookupKey: "search", Fingerprint: fingerprint("d"), Version: 1}, }, }}} @@ -90,7 +98,12 @@ func TestStoreUpdatesWithRemoteVersions(t *testing.T) { {ResourceKind: syncdomain.KindSkill, ProjectKey: "project", LookupKey: "old", Fingerprint: fingerprint("b"), Version: 2}, }} next := Manifest{Resources: []Resource{ - {ResourceKind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/existing", Fingerprint: fingerprint("c"), Version: 3}, + { + ResourceKind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/existing", + Fingerprint: fingerprint("c"), Version: 3, + FingerprintSchema: syncdomain.ProjectionFingerprintSchema, ReferenceDigest: fingerprint("e"), + AdapterBehaviorVersion: syncdomain.AdapterBehaviorVersion, AdapterBehaviorDigest: syncdomain.AdapterBehaviorDigest, + }, {ResourceKind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "search", Fingerprint: fingerprint("d")}, }} @@ -100,12 +113,18 @@ func TestStoreUpdatesWithRemoteVersions(t *testing.T) { require.Len(t, client.patches, 1) assert.Equal(t, []syncapi.SyncManifestUpsert{ {ResourceKind: syncdomain.KindTool, ResourceLookupKey: "search", Fingerprint: fingerprint("d"), Version: 0}, - {ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "config/existing", Fingerprint: fingerprint("c"), Version: 3}, + { + ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "config/existing", + Fingerprint: fingerprint("c"), Version: 3, + FingerprintSchema: syncdomain.ProjectionFingerprintSchema, ReferenceDigest: fingerprint("e"), + AdapterBehaviorVersion: syncdomain.AdapterBehaviorVersion, AdapterBehaviorDigest: syncdomain.AdapterBehaviorDigest, + }, }, client.patches[0].upserts) assert.Equal(t, []syncapi.SyncManifestDeletion{{ ResourceKind: syncdomain.KindSkill, ResourceLookupKey: "old", Version: 2, }}, client.patches[0].deletions) assert.Equal(t, 4, updated.Resources[1].Version) + assert.Equal(t, syncdomain.ProjectionFingerprintSchema, updated.Resources[1].FingerprintSchema) } func TestStoreReportsOptimisticConflict(t *testing.T) { @@ -126,7 +145,8 @@ func TestStoreReportsOptimisticConflict(t *testing.T) { func TestStoreReadsBackAnUncertainPatchThatSucceeded(t *testing.T) { applied := syncapi.SyncManifest{ - Source: "git:example/repo", + FormatVersion: syncapi.SyncManifestFormatVersion, + Source: "git:example/repo", Items: []syncapi.SyncManifestResource{{ ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "config/variation", Fingerprint: fingerprint("b"), Version: 2, diff --git a/internal/sync/projection.go b/internal/sync/projection.go new file mode 100644 index 000000000..c74a391c3 --- /dev/null +++ b/internal/sync/projection.go @@ -0,0 +1,697 @@ +package sync + +import ( + "encoding/json" + "fmt" + "maps" + "slices" + "strings" +) + +const projectionFingerprintSchema = "launchdarkly.config.projection/1" + +// ProjectionFingerprintSchema is the schema of descriptor-backed fingerprints. +const ProjectionFingerprintSchema = projectionFingerprintSchema + +// AdapterBehaviorVersion changes when adapter behavior changes mapped values. +const AdapterBehaviorVersion = 2 + +// AdapterBehaviorDigest identifies the parser, path, codec, and capability rules. +const AdapterBehaviorDigest = "sha256:e0c3bdc3ed2886e480b73a079f1e3ef3cd48c99015b4624d3bb044cecb8de2bf" + +// Field identifies one LaunchDarkly value that a repository source owns. +type Field string + +const ( + FieldPrompt Field = "prompt" + FieldMessages Field = "messages" + FieldName Field = "name" + FieldModel Field = "model" + FieldModelParameters Field = "model.parameters" + FieldTools Field = "tools" + FieldSkills Field = "skills" + FieldDescription Field = "description" + FieldSchema Field = "schema" + FieldCustomParameters Field = "customParameters" + FieldTags Field = "tags" + FieldMarkdown Field = "markdown" +) + +// Capability describes how one mapped field can change. +type Capability string + +const ( + CapabilityDirect Capability = "direct" + CapabilityPatch Capability = "patch" + CapabilityReadOnly Capability = "read-only" + CapabilityReject Capability = "reject" +) + +// FieldState records the value availability and write capability of one field. +type FieldState struct { + Capability Capability + Known bool + Reason string + Code ErrorCode + File string +} + +// Projection is the mapped content of one repository source. +type Projection interface { + Kind() Kind + projection() +} + +// VariationProjection contains only the mapped fields of one variation. +type VariationProjection struct { + Mode VariationMode `json:"mode"` + Instructions *string `json:"instructions,omitempty"` + Messages *[]Message `json:"messages,omitempty"` + Name *string `json:"name,omitempty"` + Model *string `json:"model,omitempty"` + Parameters *map[string]any `json:"parameters,omitempty"` + Tools *[]string `json:"tools,omitempty"` + Skills *[]string `json:"skills,omitempty"` + FieldStates map[Field]FieldState `json:"-"` +} + +func (VariationProjection) Kind() Kind { return KindVariation } +func (VariationProjection) projection() {} + +// ToolProjection contains only the mapped fields of one tool. +type ToolProjection struct { + Description *string `json:"description,omitempty"` + Schema *map[string]any `json:"schema,omitempty"` + CustomParameters *map[string]any `json:"customParameters,omitempty"` + Tags *[]string `json:"tags,omitempty"` + FieldStates map[Field]FieldState `json:"-"` +} + +func (ToolProjection) Kind() Kind { return KindTool } +func (ToolProjection) projection() {} + +// SkillProjection contains only the mapped fields of one skill. +type SkillProjection struct { + Description *string `json:"description,omitempty"` + Markdown *string `json:"markdown,omitempty"` + FieldStates map[Field]FieldState `json:"-"` +} + +func (SkillProjection) Kind() Kind { return KindSkill } +func (SkillProjection) projection() {} + +// ProjectionFields returns the fields that a reference maps. +func ProjectionFields(reference Reference) ([]Field, error) { + fields := make([]Field, 0, len(reference.Map)) + for name := range reference.Map { + field := Field(name) + if !field.valid() { + return nil, fmt.Errorf("unsupported mapped field %q", name) + } + fields = append(fields, field) + } + slices.Sort(fields) + return fields, nil +} + +func (field Field) valid() bool { + switch field { + case FieldPrompt, FieldMessages, FieldName, FieldModel, FieldModelParameters, FieldTools, FieldSkills, + FieldDescription, FieldSchema, FieldCustomParameters, FieldTags, FieldMarkdown: + return true + default: + _, ok := ModelParameterKey(field) + return ok + } +} + +// ModelParameterKey returns the exact key from a model parameter field. +func ModelParameterKey(field Field) (string, bool) { + const prefix = `model.parameters[` + text := string(field) + if !strings.HasPrefix(text, prefix) || !strings.HasSuffix(text, "]") { + return "", false + } + quoted := text[len(prefix) : len(text)-1] + if len(quoted) < 2 || quoted[0] != '"' { + return "", false + } + var key string + if err := json.Unmarshal([]byte(quoted), &key); err != nil { + return "", false + } + return key, true +} + +// ProjectVariation selects the mapped fields from a variation. +func ProjectVariation(variation Variation, reference Reference) (VariationProjection, error) { + fields, err := ProjectionFields(reference) + if err != nil { + return VariationProjection{}, err + } + projection := VariationProjection{Mode: variation.Mode, FieldStates: make(map[Field]FieldState)} + for _, field := range fields { + projection.FieldStates[field] = FieldState{Capability: CapabilityDirect, Known: true} + switch field { + case FieldPrompt, FieldMessages: + if variation.Mode == VariationModeCompletion { + messages := slices.Clone(variation.Messages) + projection.Messages = &messages + } else { + value := variation.Instructions + projection.Instructions = &value + } + case FieldName: + value := variation.Name + projection.Name = &value + case FieldModel: + if value, ok := variation.Model["modelName"].(string); ok { + projection.Model = &value + } + case FieldModelParameters: + parameters := canonicalModelParameters(variation) + projection.Parameters = ¶meters + default: + key, ok := ModelParameterKey(field) + if !ok { + return VariationProjection{}, fmt.Errorf("field %q is not valid for a variation", field) + } + parameters := canonicalModelParameters(variation) + if parameter, present := parameters[key]; present { + if projection.Parameters == nil { + value := map[string]any{} + projection.Parameters = &value + } + (*projection.Parameters)[key] = cloneJSONValue(parameter) + } + } + } + if reference.Tools != nil { + values := referenceKeys(variation.Tools) + projection.Tools = &values + projection.FieldStates[FieldTools] = FieldState{Capability: CapabilityDirect, Known: true} + } + if reference.Skills != nil { + values := referenceKeys(variation.Skills) + projection.Skills = &values + projection.FieldStates[FieldSkills] = FieldState{Capability: CapabilityDirect, Known: true} + } + return projection, nil +} + +// ApplyVariationProjection merges mapped fields into a variation. +func ApplyVariationProjection(variation Variation, projection VariationProjection) Variation { + if projection.Mode.Valid() { + variation.Mode = projection.Mode + } + if projection.Instructions != nil { + variation.Instructions, variation.Messages = *projection.Instructions, nil + } + if projection.Messages != nil { + variation.Instructions, variation.Messages = "", slices.Clone(*projection.Messages) + } + if projection.Name != nil { + variation.Name = *projection.Name + } + if projection.Model != nil { + variation.Model = cloneMap(variation.Model) + variation.Model["modelName"] = *projection.Model + } + if projection.Parameters != nil { + variation.Model = cloneMap(variation.Model) + if _, whole := projection.FieldStates[FieldModelParameters]; whole { + variation.Model["parameters"] = cloneMap(*projection.Parameters) + } else { + parameters, _ := variation.Model["parameters"].(map[string]any) + parameters = cloneMap(parameters) + for _, field := range projectionStateFields(projection) { + key, parameter := ModelParameterKey(field) + if !parameter { + continue + } + if value, present := (*projection.Parameters)[key]; present { + parameters[key] = cloneJSONValue(value) + } + } + variation.Model["parameters"] = parameters + } + } + if projection.Tools != nil { + variation.Tools = attachmentReferences(*projection.Tools) + } + if projection.Skills != nil { + variation.Skills = attachmentReferences(*projection.Skills) + } + variation.Model = CanonicalModel(variation.Model, variation.Tools) + return variation +} + +// ProjectAttachment selects the mapped fields from a tool or skill. +func ProjectAttachment(attachment Attachment, reference Reference) (Projection, error) { + fields, err := ProjectionFields(reference) + if err != nil { + return nil, err + } + switch { + case attachment.Tool != nil: + projection := ToolProjection{FieldStates: make(map[Field]FieldState)} + for _, field := range fields { + projection.FieldStates[field] = FieldState{Capability: CapabilityDirect, Known: true} + switch field { + case FieldDescription: + projection.Description = attachment.Tool.Description + case FieldSchema: + value := cloneMap(attachment.Tool.Schema) + projection.Schema = &value + case FieldCustomParameters: + value := cloneMap(attachment.Tool.CustomParameters) + projection.CustomParameters = &value + case FieldTags: + value := slices.Clone(attachment.Tool.Tags) + slices.Sort(value) + value = slices.Compact(value) + projection.Tags = &value + default: + return nil, fmt.Errorf("field %q is not valid for a tool", field) + } + } + return projection, nil + case attachment.Skill != nil: + projection := SkillProjection{FieldStates: make(map[Field]FieldState)} + for _, field := range fields { + projection.FieldStates[field] = FieldState{Capability: CapabilityDirect, Known: true} + switch field { + case FieldDescription: + value := attachment.Skill.Description + projection.Description = &value + case FieldMarkdown: + value := attachment.Skill.Markdown + projection.Markdown = &value + default: + return nil, fmt.Errorf("field %q is not valid for a skill", field) + } + } + return projection, nil + default: + return nil, fmt.Errorf("%s %q has no content", attachment.Kind, attachment.Key()) + } +} + +// ApplyAttachmentProjection merges mapped fields into an attachment. +func ApplyAttachmentProjection(attachment Attachment, projection Projection) (Attachment, error) { + switch value := projection.(type) { + case ToolProjection: + if attachment.Tool == nil { + return Attachment{}, fmt.Errorf("tool %q has no content", attachment.Key()) + } + tool := *attachment.Tool + if value.Description != nil { + description := *value.Description + tool.Description = &description + } + if value.Schema != nil { + tool.Schema = cloneMap(*value.Schema) + } + if value.CustomParameters != nil { + tool.CustomParameters = cloneMap(*value.CustomParameters) + } + if value.Tags != nil { + tool.Tags = slices.Clone(*value.Tags) + slices.Sort(tool.Tags) + tool.Tags = slices.Compact(tool.Tags) + } + attachment.Tool = &tool + case SkillProjection: + if attachment.Skill == nil { + return Attachment{}, fmt.Errorf("skill %q has no content", attachment.Key()) + } + skill := *attachment.Skill + if value.Description != nil { + skill.Description = *value.Description + } + if value.Markdown != nil { + skill.Markdown = *value.Markdown + } + attachment.Skill = &skill + default: + return Attachment{}, fmt.Errorf("unsupported projection %T", projection) + } + return attachment, nil +} + +// FingerprintProjection returns a stable fingerprint for mapped fields only. +func FingerprintProjection(id ResourceID, projection Projection) (string, error) { + return fingerprint("projection", struct { + Schema string `json:"schema"` + ResourceKind Kind `json:"resourceKind"` + ProjectKey string `json:"projectKey"` + LookupKey string `json:"lookupKey"` + Projection Projection `json:"projection"` + }{ + Schema: projectionFingerprintSchema, + ResourceKind: id.Kind, + ProjectKey: id.ProjectKey, + LookupKey: id.LookupKey, + Projection: projection, + }) +} + +// ReferenceDigest returns a stable digest of one editable descriptor. +func ReferenceDigest(reference Reference) (string, error) { + return fingerprint("reference descriptor", struct { + Schema string `json:"schema"` + File string `json:"file"` + Format string `json:"format"` + Select string `json:"select,omitempty"` + Map map[string]string `json:"map,omitempty"` + }{ + Schema: "launchdarkly.config.reference/1", + File: reference.File, Format: reference.Format, Select: reference.Select, Map: reference.Map, + }) +} + +func referenceKeys(values []AttachmentRef) []string { + keys := make([]string, len(values)) + for index := range values { + keys[index] = values[index].Key + } + slices.Sort(keys) + return keys +} + +func attachmentReferences(keys []string) []AttachmentRef { + references := make([]AttachmentRef, len(keys)) + for index := range keys { + references[index] = AttachmentRef{Key: keys[index]} + } + return references +} + +func cloneMap(source map[string]any) map[string]any { + if source == nil { + return map[string]any{} + } + data, _ := json.Marshal(source) + var result map[string]any + _ = json.Unmarshal(data, &result) + return result +} + +func cloneJSONValue(source any) any { + data, err := json.Marshal(source) + if err != nil { + return source + } + var result any + if err := json.Unmarshal(data, &result); err != nil { + return source + } + return result +} + +func canonicalModelParameters(variation Variation) map[string]any { + model := CanonicalModel(variation.Model, variation.Tools) + parameters, _ := model["parameters"].(map[string]any) + return cloneMap(parameters) +} + +// ProjectionFieldState returns one mapped field capability. +func ProjectionFieldState(projection Projection, field Field) (FieldState, bool) { + var states map[Field]FieldState + switch value := projection.(type) { + case VariationProjection: + states = value.FieldStates + case ToolProjection: + states = value.FieldStates + case SkillProjection: + states = value.FieldStates + } + state, ok := states[field] + return state, ok +} + +// ProjectionFieldStates returns a copy of every mapped field state. +func ProjectionFieldStates(projection Projection) map[Field]FieldState { + result := map[Field]FieldState{} + for _, field := range projectionStateFields(projection) { + state, _ := ProjectionFieldState(projection, field) + result[field] = state + } + return result +} + +// ProjectionFieldValue returns one known mapped value. +func ProjectionFieldValue(projection Projection, field Field) (any, bool) { + switch value := projection.(type) { + case VariationProjection: + switch field { + case FieldPrompt, FieldMessages: + if value.Instructions != nil { + return *value.Instructions, true + } + if value.Messages != nil { + return *value.Messages, true + } + case FieldName: + if value.Name != nil { + return *value.Name, true + } + case FieldModel: + if value.Model != nil { + return *value.Model, true + } + case FieldModelParameters: + if value.Parameters != nil { + return cloneMap(*value.Parameters), true + } + case FieldTools: + if value.Tools != nil { + return *value.Tools, true + } + case FieldSkills: + if value.Skills != nil { + return *value.Skills, true + } + default: + if key, ok := ModelParameterKey(field); ok && value.Parameters != nil { + parameter, present := (*value.Parameters)[key] + return cloneJSONValue(parameter), present + } + } + case ToolProjection: + switch field { + case FieldDescription: + if value.Description != nil { + return *value.Description, true + } + case FieldSchema: + if value.Schema != nil { + return *value.Schema, true + } + case FieldCustomParameters: + if value.CustomParameters != nil { + return *value.CustomParameters, true + } + case FieldTags: + if value.Tags != nil { + return *value.Tags, true + } + } + case SkillProjection: + if field == FieldDescription && value.Description != nil { + return *value.Description, true + } + if field == FieldMarkdown && value.Markdown != nil { + return *value.Markdown, true + } + } + return nil, false +} + +// FillReadOnlyFields copies server values for local fields that cannot be read. +func FillReadOnlyFields(local, server Projection) (Projection, error) { + result := local + for _, field := range projectionStateFields(local) { + state, _ := ProjectionFieldState(local, field) + if state.Capability != CapabilityReadOnly || state.Known { + continue + } + if value, ok := ProjectionFieldValue(server, field); ok { + var err error + result, err = setProjectionField(result, field, value) + if err != nil { + return nil, err + } + result = setProjectionState(result, field, state) + } + } + return result, nil +} + +func setProjectionState(projection Projection, field Field, state FieldState) Projection { + switch value := projection.(type) { + case VariationProjection: + value.FieldStates[field] = state + return value + case ToolProjection: + value.FieldStates[field] = state + return value + case SkillProjection: + value.FieldStates[field] = state + return value + default: + return projection + } +} + +// ChangedProjection returns only server fields whose projected values differ. +func ChangedProjection(local, server Projection) (Projection, error) { + result := emptyProjection(server) + for _, field := range projectionStateFields(local) { + left, leftOK := ProjectionFieldValue(local, field) + right, rightOK := ProjectionFieldValue(server, field) + leftJSON, _ := json.Marshal(left) + rightJSON, _ := json.Marshal(right) + if leftOK == rightOK && string(leftJSON) == string(rightJSON) { + continue + } + if rightOK { + var err error + result, err = setProjectionField(result, field, right) + if err != nil { + return nil, err + } + } + } + return result, nil +} + +// ProjectionIncludes reports whether actual has every known value in expected. +func ProjectionIncludes(actual, expected Projection) bool { + for _, field := range projectionStateFields(expected) { + want, wantOK := ProjectionFieldValue(expected, field) + have, haveOK := ProjectionFieldValue(actual, field) + wantJSON, _ := json.Marshal(want) + haveJSON, _ := json.Marshal(have) + if wantOK != haveOK || string(wantJSON) != string(haveJSON) { + return false + } + } + return true +} + +func projectionStateFields(projection Projection) []Field { + var states map[Field]FieldState + switch value := projection.(type) { + case VariationProjection: + states = value.FieldStates + case ToolProjection: + states = value.FieldStates + case SkillProjection: + states = value.FieldStates + } + fields := slices.Collect(maps.Keys(states)) + slices.Sort(fields) + return fields +} + +func emptyProjection(projection Projection) Projection { + switch value := projection.(type) { + case VariationProjection: + return VariationProjection{Mode: value.Mode, FieldStates: make(map[Field]FieldState)} + case ToolProjection: + return ToolProjection{FieldStates: make(map[Field]FieldState)} + case SkillProjection: + return SkillProjection{FieldStates: make(map[Field]FieldState)} + default: + return projection + } +} + +func setProjectionField(projection Projection, field Field, raw any) (Projection, error) { + state := FieldState{Capability: CapabilityDirect, Known: true} + switch value := projection.(type) { + case VariationProjection: + if value.FieldStates == nil { + value.FieldStates = make(map[Field]FieldState) + } + value.FieldStates[field] = state + switch field { + case FieldPrompt, FieldMessages: + switch prompt := raw.(type) { + case string: + value.Instructions = &prompt + case []Message: + value.Messages = &prompt + } + case FieldName: + item := raw.(string) + value.Name = &item + case FieldModel: + item := raw.(string) + value.Model = &item + case FieldModelParameters: + item, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("field %q needs an object", field) + } + item = cloneMap(item) + value.Parameters = &item + case FieldTools: + item := slices.Clone(raw.([]string)) + value.Tools = &item + case FieldSkills: + item := slices.Clone(raw.([]string)) + value.Skills = &item + default: + key, ok := ModelParameterKey(field) + if !ok { + return nil, fmt.Errorf("field %q is not valid for a variation", field) + } + if value.Parameters == nil { + parameters := map[string]any{} + value.Parameters = ¶meters + } + (*value.Parameters)[key] = cloneJSONValue(raw) + } + return value, nil + case ToolProjection: + if value.FieldStates == nil { + value.FieldStates = make(map[Field]FieldState) + } + value.FieldStates[field] = state + switch field { + case FieldDescription: + item := raw.(string) + value.Description = &item + case FieldSchema: + item := cloneMap(raw.(map[string]any)) + value.Schema = &item + case FieldCustomParameters: + item := cloneMap(raw.(map[string]any)) + value.CustomParameters = &item + case FieldTags: + item := slices.Clone(raw.([]string)) + value.Tags = &item + default: + return nil, fmt.Errorf("field %q is not valid for a tool", field) + } + return value, nil + case SkillProjection: + if value.FieldStates == nil { + value.FieldStates = make(map[Field]FieldState) + } + value.FieldStates[field] = state + item := raw.(string) + if field == FieldDescription { + value.Description = &item + } else if field == FieldMarkdown { + value.Markdown = &item + } else { + return nil, fmt.Errorf("field %q is not valid for a skill", field) + } + return value, nil + default: + return nil, fmt.Errorf("unsupported projection %T", projection) + } +} diff --git a/internal/sync/projection_parameters_test.go b/internal/sync/projection_parameters_test.go new file mode 100644 index 000000000..fa6842289 --- /dev/null +++ b/internal/sync/projection_parameters_test.go @@ -0,0 +1,170 @@ +package sync + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestModelParameterKeyPreservesExactKeys(t *testing.T) { + tests := []struct { + target string + key string + valid bool + }{ + {target: `model.parameters["temperature"]`, key: "temperature", valid: true}, + {target: `model.parameters["top_k"]`, key: "top_k", valid: true}, + {target: `model.parameters["top_p"]`, key: "top_p", valid: true}, + {target: `model.parameters["max_tokens"]`, key: "max_tokens", valid: true}, + {target: `model.parameters["Provider.Future-Key"]`, key: "Provider.Future-Key", valid: true}, + {target: `model.parameters["Case\u0053ensitive"]`, key: "CaseSensitive", valid: true}, + {target: `model.parameters[temperature]`}, + {target: `model.parameters.temperature`}, + {target: `model.parameters["temperature"].nested`}, + } + for _, test := range tests { + t.Run(test.target, func(t *testing.T) { + key, valid := ModelParameterKey(Field(test.target)) + assert.Equal(t, test.valid, valid) + assert.Equal(t, test.key, key) + }) + } +} + +func TestProjectVariationPreservesJSONModelParameterValues(t *testing.T) { + tests := []struct { + name string + key string + value any + }{ + {name: "temperature", key: "temperature", value: 0.2}, + {name: "top k", key: "top_k", value: 40.0}, + {name: "top p", key: "top_p", value: 0.9}, + {name: "max tokens", key: "max_tokens", value: 1024.0}, + {name: "future provider key", key: "Provider.Future-Key", value: "fast"}, + {name: "string", key: "response_style", value: "brief"}, + {name: "boolean", key: "reasoning", value: true}, + {name: "null", key: "seed", value: nil}, + {name: "nested object", key: "provider", value: map[string]any{"mode": "fast", "retry": true}}, + {name: "array", key: "stop", value: []any{"END", map[string]any{"token": 2.0}}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + target := Field(`model.parameters["` + test.key + `"]`) + variation := Variation{ + Mode: VariationModeAgent, + Model: map[string]any{"parameters": map[string]any{test.key: test.value}}, + } + projection, err := ProjectVariation(variation, Reference{ + Map: map[string]string{string(target): "value"}, + }) + require.NoError(t, err) + require.NotNil(t, projection.Parameters) + assert.Equal(t, test.value, (*projection.Parameters)[test.key]) + value, present := ProjectionFieldValue(projection, target) + assert.True(t, present) + assert.Equal(t, test.value, value) + }) + } +} + +func TestApplyVariationProjectionUsesWholeOrIndividualParameterOwnership(t *testing.T) { + base := Variation{ + Mode: VariationModeAgent, + Model: map[string]any{"parameters": map[string]any{ + "temperature": 0.2, + "top_k": 20.0, + "keep": true, + }}, + } + tests := []struct { + name string + projection VariationProjection + want map[string]any + }{ + { + name: "individual mapping merges one key", + projection: VariationProjection{ + Parameters: &map[string]any{"top_k": 50.0}, + FieldStates: map[Field]FieldState{`model.parameters["top_k"]`: {Known: true}}, + }, + want: map[string]any{"temperature": 0.2, "top_k": 50.0, "keep": true}, + }, + { + name: "individual mapping preserves null", + projection: VariationProjection{ + Parameters: &map[string]any{"top_p": nil}, + FieldStates: map[Field]FieldState{`model.parameters["top_p"]`: {Known: true}}, + }, + want: map[string]any{"temperature": 0.2, "top_k": 20.0, "top_p": nil, "keep": true}, + }, + { + name: "whole mapping replaces canonical parameters", + projection: VariationProjection{ + Parameters: &map[string]any{"max_tokens": 512.0}, + FieldStates: map[Field]FieldState{FieldModelParameters: {Known: true}}, + }, + want: map[string]any{"max_tokens": 512.0}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + applied := ApplyVariationProjection(base, test.projection) + assert.Equal(t, test.want, applied.Model["parameters"]) + }) + } +} + +func TestModelParameterProjectionChangeAndFingerprint(t *testing.T) { + id := VariationID("project", "config", "default") + target := Field(`model.parameters["temperature"]`) + state := map[Field]FieldState{target: {Capability: CapabilityDirect, Known: true}} + localParameters := map[string]any{"temperature": 0.2} + serverParameters := map[string]any{"temperature": 0.7} + local := VariationProjection{Mode: VariationModeAgent, Parameters: &localParameters, FieldStates: state} + server := VariationProjection{Mode: VariationModeAgent, Parameters: &serverParameters, FieldStates: state} + + localFingerprint, err := FingerprintProjection(id, local) + require.NoError(t, err) + serverFingerprint, err := FingerprintProjection(id, server) + require.NoError(t, err) + assert.NotEqual(t, localFingerprint, serverFingerprint) + + changed, err := ChangedProjection(local, server) + require.NoError(t, err) + value, present := ProjectionFieldValue(changed, target) + assert.True(t, present) + assert.Equal(t, 0.7, value) + assert.False(t, ProjectionIncludes(local, server)) + assert.True(t, ProjectionIncludes(server, changed)) +} + +func TestModelParameterProjectionExcludesExpandedTools(t *testing.T) { + variation := Variation{ + Mode: VariationModeAgent, + Tools: []AttachmentRef{{Key: "search"}}, + Model: map[string]any{"parameters": map[string]any{ + "temperature": 0.2, + "tools": []any{map[string]any{"name": "search"}}, + }}, + } + projection, err := ProjectVariation(variation, Reference{ + Map: map[string]string{string(FieldModelParameters): "parameters"}, + }) + require.NoError(t, err) + require.NotNil(t, projection.Parameters) + assert.Equal(t, map[string]any{"temperature": 0.2}, *projection.Parameters) + + parameters := map[string]any{ + "temperature": 0.3, + "tools": []any{map[string]any{"name": "search"}}, + } + applied := ApplyVariationProjection(variation, VariationProjection{ + Parameters: ¶meters, + FieldStates: map[Field]FieldState{ + FieldModelParameters: {Known: true}, + }, + }) + assert.Equal(t, map[string]any{"temperature": 0.3}, applied.Model["parameters"]) +} diff --git a/internal/sync/prompt/acceptance_test.go b/internal/sync/prompt/acceptance_test.go index 3da41cbe9..050de741d 100644 --- a/internal/sync/prompt/acceptance_test.go +++ b/internal/sync/prompt/acceptance_test.go @@ -1960,7 +1960,7 @@ func runScoped(t *testing.T, root string, api *directAPI, options syncprompt.Opt func runWith(t *testing.T, root string, api *directAPI, runner syncprompt.Runner, options syncprompt.Options) (string, string, error) { t.Helper() if manifestsByRoot[root] == nil { - manifestsByRoot[root] = &syncapi.SyncManifest{Source: "git:example/repo", Items: []syncapi.SyncManifestResource{}} + manifestsByRoot[root] = &syncapi.SyncManifest{FormatVersion: syncapi.SyncManifestFormatVersion, Source: "git:example/repo", Items: []syncapi.SyncManifestResource{}} } api.manifest = manifestsByRoot[root] var output, notes bytes.Buffer @@ -2273,7 +2273,7 @@ func runPrompt(t *testing.T, root string, client resources.Client, arguments ... t.Setenv("XDG_CONFIG_HOME", t.TempDir()) manifest := manifestsByRoot[root] if manifest == nil { - manifest = &syncapi.SyncManifest{Source: "git:example/repo", Items: []syncapi.SyncManifestResource{}} + manifest = &syncapi.SyncManifest{FormatVersion: syncapi.SyncManifestFormatVersion, Source: "git:example/repo", Items: []syncapi.SyncManifestResource{}} manifestsByRoot[root] = manifest } switch api := client.(type) { @@ -2436,7 +2436,7 @@ func writeRemoteManifest(t *testing.T, root string, values ...syncdomain.Variati Kind: syncdomain.KindVariation, ProjectKey: "production", LookupKey: "support/" + value.Key, }, fingerprint(t, value)) } - remote := &syncapi.SyncManifest{Source: "git:example/repo"} + remote := &syncapi.SyncManifest{FormatVersion: syncapi.SyncManifestFormatVersion, Source: "git:example/repo"} for _, resource := range manifest.Resources { remote.Items = append(remote.Items, syncapi.SyncManifestResource{ ResourceKind: resource.ResourceKind, diff --git a/internal/sync/relationship_projection_test.go b/internal/sync/relationship_projection_test.go new file mode 100644 index 000000000..907ec918e --- /dev/null +++ b/internal/sync/relationship_projection_test.go @@ -0,0 +1,76 @@ +package sync + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReferenceRelationshipsPreserveOwnershipAndExactKeys(t *testing.T) { + empty := []AttachmentRef{} + keys := []AttachmentRef{{Key: "Tool.Case_sensitive"}} + variation := Variation{ + Mode: VariationModeAgent, + Tools: []AttachmentRef{{Key: "Tool.Case_sensitive", Version: 9}}, + } + + unowned, err := ProjectVariation(variation, Reference{}) + require.NoError(t, err) + assert.Nil(t, unowned.Tools) + _, owned := unowned.FieldStates[FieldTools] + assert.False(t, owned) + + ownedEmpty, err := ProjectVariation(variation, Reference{Tools: &empty}) + require.NoError(t, err) + require.NotNil(t, ownedEmpty.Tools) + assert.Equal(t, []string{"Tool.Case_sensitive"}, *ownedEmpty.Tools) + + ownedKeys, err := ProjectVariation(variation, Reference{Tools: &keys}) + require.NoError(t, err) + require.NotNil(t, ownedKeys.Tools) + assert.Equal(t, []string{"Tool.Case_sensitive"}, *ownedKeys.Tools) + + emptyKeys := []string{} + applied := ApplyVariationProjection(Variation{}, VariationProjection{ + Tools: &emptyKeys, + FieldStates: map[Field]FieldState{ + FieldTools: {Capability: CapabilityDirect, Known: true}, + }, + }) + require.NotNil(t, applied.Tools) + assert.Empty(t, applied.Tools) + applied = ApplyVariationProjection(Variation{}, ownedKeys) + assert.Equal(t, []AttachmentRef{{Key: "Tool.Case_sensitive"}}, applied.Tools) +} + +func TestReferenceDigestExcludesRelationships(t *testing.T) { + first := []AttachmentRef{{Key: "search"}} + second := []AttachmentRef{{Key: "lookup"}} + base := Reference{File: "prompt.md", Format: "markdown"} + + left, err := ReferenceDigest(Reference{ + File: base.File, Format: base.Format, Tools: &first, + }) + require.NoError(t, err) + right, err := ReferenceDigest(Reference{ + File: base.File, Format: base.Format, Tools: &second, + }) + require.NoError(t, err) + assert.Equal(t, left, right) +} + +func TestValidateAttachmentRefsAllowsCompletionToolsAndRejectsSkills(t *testing.T) { + require.NoError(t, ValidateAttachmentRefs( + VariationModeCompletion, + []AttachmentRef{{Key: "search"}}, + nil, + )) + + err := ValidateAttachmentRefs( + VariationModeCompletion, + nil, + []AttachmentRef{{Key: "support"}}, + ) + require.EqualError(t, err, "skills can only be attached to agent-mode configs") +} diff --git a/internal/sync/resource.go b/internal/sync/resource.go index 5dcfb77ea..b80743366 100644 --- a/internal/sync/resource.go +++ b/internal/sync/resource.go @@ -22,6 +22,22 @@ const ( KindSkill Kind = "skill" ) +// SyncDirection is the LaunchDarkly setting that controls allowed writes. +type SyncDirection string + +const ( + SyncDirectionCodeCanonical SyncDirection = "code_canonical" + SyncDirectionServerCanonical SyncDirection = "server_canonical" + SyncDirectionBoth SyncDirection = "both" +) + +// Valid reports whether LaunchDarkly returned a supported sync direction. +func (direction SyncDirection) Valid() bool { + return direction == SyncDirectionCodeCanonical || + direction == SyncDirectionServerCanonical || + direction == SyncDirectionBoth +} + // ResourceID identifies one synchronized resource. // // For a variation, LookupKey is "config-key/variation-key". For a tool or a @@ -143,11 +159,23 @@ func ValidateCreate(kind Kind, key, name, description string) error { } } -// Reference links a variation to a prompt file elsewhere in the repository. -// Format names the adapter that reads and writes the file. +// Reference links one resource to a file elsewhere in the repository. +// Select identifies one value in the file. Map connects resource fields to +// paths below that value. type Reference struct { - File string `yaml:"file"` - Format string `yaml:"format"` + File string `json:"file" yaml:"file"` + Format string `json:"format" yaml:"format"` + Select string `json:"select,omitempty" yaml:"select,omitempty"` + Map map[string]string `json:"map,omitempty" yaml:"map,omitempty"` + Tools *[]AttachmentRef `json:"tools,omitempty" yaml:"tools,omitempty"` + Skills *[]AttachmentRef `json:"skills,omitempty" yaml:"skills,omitempty"` +} + +// IndexedSelection identifies one indexed item in a repository source. +type IndexedSelection struct { + Path string `yaml:"path"` + CollectionLength int `yaml:"collectionLength"` + Fingerprint string `yaml:"fingerprint"` } // SyncedResource is one variation compiled from the local workspace. @@ -156,8 +184,13 @@ type SyncedResource struct { ProjectKey string LookupKey string // Upsert lets sync create the variation in LaunchDarkly when it is absent. - Upsert bool - Variation Variation + Upsert bool + // Source is the descriptor-backed repository source, or nil for a managed file. + Source *Reference + Dependencies []string + IndexedSelections []IndexedSelection + Projection Projection + Variation Variation } // ID returns the identity of the compiled resource. diff --git a/internal/sync/variation.go b/internal/sync/variation.go index 22b853045..f92cd51aa 100644 --- a/internal/sync/variation.go +++ b/internal/sync/variation.go @@ -82,8 +82,6 @@ func (variation Variation) Validate() error { return errors.New("key is required") case variation.Name == "": return errors.New("name is required") - case variation.Mode != VariationModeAgent && len(variation.Skills) != 0: - return errors.New("skills can only be attached to agent-mode configs") } // LaunchDarkly drops a value of these keys that is not an object. @@ -95,9 +93,22 @@ func (variation Variation) Validate() error { } } + return ValidateAttachmentRefs(variation.Mode, variation.Tools, variation.Skills) +} + +// ValidateAttachmentRefs makes sure that relationship keys are usable for the +// variation mode. +func ValidateAttachmentRefs(mode VariationMode, tools, skills []AttachmentRef) error { + if mode != VariationModeAgent && len(skills) != 0 { + return errors.New("skills can only be attached to agent-mode configs") + } for _, kind := range AttachmentKinds { seen := make(map[string]struct{}) - for _, ref := range variation.Refs(kind) { + refs := tools + if kind == AttachmentSkill { + refs = skills + } + for _, ref := range refs { if strings.TrimSpace(ref.Key) == "" { return fmt.Errorf("%s key is required", kind) } From f17e19bb9e641f34e34916267bd43f4b1d9a3858 Mon Sep 17 00:00:00 2001 From: Cliff Tawiah Date: Sun, 1 Nov 2026 20:42:29 -0500 Subject: [PATCH 3/5] feat(sync): add reference registry and Markdown adapters --- .../reference/adapter_conformance_test.go | 135 +++ internal/sync/reference/document.go | 389 +++++++ internal/sync/reference/fixture_test.go | 25 + internal/sync/reference/path.go | 270 +++++ internal/sync/reference/registry.go | 119 ++ .../sync/reference/registry_external_test.go | 42 + .../sync/reference/repository_paths_test.go | 61 + internal/sync/reference/source.go | 1027 +++++++++++++++++ internal/sync/reference/source_test.go | 120 ++ .../agent-skill/refunds/references/policy.md | 3 + .../refunds/scripts/check_refund.py | 2 + .../refunds/skill-instructions.md.golden | 12 + .../markdown/agents-instructions.md.golden | 9 + 13 files changed, 2214 insertions(+) create mode 100644 internal/sync/reference/adapter_conformance_test.go create mode 100644 internal/sync/reference/document.go create mode 100644 internal/sync/reference/fixture_test.go create mode 100644 internal/sync/reference/path.go create mode 100644 internal/sync/reference/registry.go create mode 100644 internal/sync/reference/registry_external_test.go create mode 100644 internal/sync/reference/repository_paths_test.go create mode 100644 internal/sync/reference/source.go create mode 100644 internal/sync/reference/source_test.go create mode 100644 internal/sync/reference/testdata/agent-skill/refunds/references/policy.md create mode 100644 internal/sync/reference/testdata/agent-skill/refunds/scripts/check_refund.py create mode 100644 internal/sync/reference/testdata/agent-skill/refunds/skill-instructions.md.golden create mode 100644 internal/sync/reference/testdata/markdown/agents-instructions.md.golden diff --git a/internal/sync/reference/adapter_conformance_test.go b/internal/sync/reference/adapter_conformance_test.go new file mode 100644 index 000000000..22246424c --- /dev/null +++ b/internal/sync/reference/adapter_conformance_test.go @@ -0,0 +1,135 @@ +package reference + +import ( + "bytes" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestRegisteredFormatsReadAndWriteRealisticSources(t *testing.T) { + tests := []struct { + name string + fixture string + realPath string + format string + id syncdomain.ResourceID + selectPath string + fieldMap map[string]string + want string + update string + }{ + { + name: "Markdown section", fixture: "markdown/agents-instructions.md.golden", format: "markdown", + id: syncdomain.VariationID("project", "support", "default"), + selectPath: `sections[heading="Support agent"]`, fieldMap: map[string]string{"prompt": "body"}, + want: "Ask for an order ID before discussing an order.", update: "Ask for an order ID and region.", + }, + { + name: "Agent Skill", fixture: "agent-skill/refunds/skill-instructions.md.golden", + realPath: ".agents/skills/refunds/SKILL.md", format: "agent-skill", + id: syncdomain.ResourceID{Kind: syncdomain.KindSkill, ProjectKey: "project", LookupKey: "refunds"}, + want: "Explain the refund policy and escalation rules.", update: "Explain refunds and escalation rules.", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source := fixtureFile(t, test.fixture) + documentPath := test.fixture + if test.realPath != "" { + documentPath, source = materializeFixture(t, test.fixture, test.realPath) + } + reference := syncdomain.Reference{ + File: documentPath, Format: test.format, Select: test.selectPath, Map: test.fieldMap, + } + projection, err := ParseSource(sourceBundle(documentPath, source), test.id, syncdomain.VariationModeAgent, reference) + require.NoError(t, err) + assert.Equal(t, test.want, primaryText(t, projection)) + + desired := projectionWithText(projection, test.update) + result, err := UpdateSource(sourceBundle(documentPath, source), test.id, syncdomain.VariationModeAgent, reference, desired) + require.NoError(t, err) + edits := directEdits(t, result) + updated := applyConformanceEdits(t, source, edits) + reparsed, err := ParseSource(sourceBundle(documentPath, updated), test.id, syncdomain.VariationModeAgent, reference) + require.NoError(t, err) + assert.Equal(t, test.update, primaryText(t, reparsed)) + }) + } +} + +func sourceBundle(path string, data []byte) SourceBundle { + return SourceBundle{Primary: Document{Path: path, Data: data}} +} + +func primaryText(t *testing.T, projection syncdomain.Projection) string { + t.Helper() + switch value := projection.(type) { + case syncdomain.VariationProjection: + require.NotNil(t, value.Instructions) + return *value.Instructions + case syncdomain.ToolProjection: + require.NotNil(t, value.Description) + return *value.Description + case syncdomain.SkillProjection: + require.NotNil(t, value.Description) + return *value.Description + default: + t.Fatalf("unsupported projection %T", projection) + return "" + } +} + +func projectionWithText(projection syncdomain.Projection, text string) syncdomain.Projection { + switch value := projection.(type) { + case syncdomain.VariationProjection: + value.Instructions = &text + value.Messages = nil + return value + case syncdomain.ToolProjection: + value.Description = &text + return value + case syncdomain.SkillProjection: + value.Description = &text + return value + default: + return projection + } +} + +func directEdits(t *testing.T, result UpdateResult) []TextEdit { + t.Helper() + var edits []TextEdit + for field, update := range result.Fields { + require.Equal(t, CapabilityDirect, update.Capability, field) + edits = append(edits, update.Edits...) + } + require.NotEmpty(t, edits) + return edits +} + +func applyConformanceEdits(t *testing.T, source []byte, edits []TextEdit) []byte { + t.Helper() + slices.SortFunc(edits, func(left, right TextEdit) int { return right.Start - left.Start }) + updated := bytes.Clone(source) + for _, edit := range edits { + require.Equal(t, edit.Expected, source[edit.Start:edit.End]) + updated = append(append(append([]byte{}, updated[:edit.Start]...), edit.Replacement...), updated[edit.End:]...) + } + ascending := slices.Clone(edits) + slices.SortFunc(ascending, func(left, right TextEdit) int { return left.Start - right.Start }) + beforeOffset, afterOffset := 0, 0 + for _, edit := range ascending { + length := edit.Start - beforeOffset + require.Equal(t, source[beforeOffset:edit.Start], updated[afterOffset:afterOffset+length]) + beforeOffset = edit.End + afterOffset += length + len(edit.Replacement) + } + require.Equal(t, source[beforeOffset:], updated[afterOffset:]) + return updated +} diff --git a/internal/sync/reference/document.go b/internal/sync/reference/document.go new file mode 100644 index 000000000..3e888d854 --- /dev/null +++ b/internal/sync/reference/document.go @@ -0,0 +1,389 @@ +package reference + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "unicode" + "unicode/utf8" + + "gopkg.in/yaml.v3" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type markdownAdapter struct{} + +func (markdownAdapter) Format() string { return "markdown" } +func (markdownAdapter) Inspect(bundle SourceBundle) (Inspection, error) { + root, err := parseMarkdown(bundle.Primary) + return Inspection{root: root}, err +} +func (markdownAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + if kind == syncdomain.KindVariation { + return "$", map[string]string{string(syncdomain.FieldPrompt): "$"} + } + return "$", nil +} + +type agentSkillAdapter struct{} + +func (agentSkillAdapter) Format() string { return "agent-skill" } +func (agentSkillAdapter) Inspect(bundle SourceBundle) (Inspection, error) { + root, err := parseAgentSkill(bundle.Primary) + return Inspection{root: root}, err +} +func (agentSkillAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + if kind == syncdomain.KindSkill { + return "$", map[string]string{ + string(syncdomain.FieldDescription): "frontmatter.description", + string(syncdomain.FieldMarkdown): "body", + } + } + return "$", nil +} + +func init() { + mustRegister(markdownAdapter{}) + mustRegister(agentSkillAdapter{}) +} + +func parseMarkdown(document Document) (*sourceNode, error) { + if !utf8.Valid(document.Data) { + return nil, fmt.Errorf("%s: source is not valid UTF-8", document.Path) + } + data := document.Data + root := markdownTextNode(data, 0, len(data)) + root.fields = map[string]*sourceNode{"body": root} + root.fields["sections"] = &sourceNode{items: markdownSections(data), collection: true} + return root, nil +} + +func markdownSections(data []byte) []*sourceNode { + type heading struct { + level int + text string + lineStart int + bodyStart int + } + var headings []heading + var fenceCharacter byte + fenceLength := 0 + for offset := 0; offset < len(data); { + end := bytes.IndexByte(data[offset:], '\n') + if end < 0 { + end = len(data) + } else { + end += offset + 1 + } + line := strings.TrimSuffix(strings.TrimSuffix(string(data[offset:end]), "\n"), "\r") + trimmed := strings.TrimLeft(line, " ") + indent := len(line) - len(trimmed) + if indent > 3 || strings.HasPrefix(trimmed, "\t") { + offset = end + continue + } + marker, markerLength := markdownFence(trimmed) + switch { + case fenceLength == 0 && markerLength >= 3: + fenceCharacter, fenceLength = marker, markerLength + case fenceLength != 0 && marker == fenceCharacter && markerLength >= fenceLength: + fenceCharacter, fenceLength = 0, 0 + case fenceLength == 0: + level := 0 + for level < len(trimmed) && trimmed[level] == '#' { + level++ + } + if level > 0 && level <= 6 && level < len(trimmed) && trimmed[level] == ' ' { + text := markdownHeadingText(trimmed[level:]) + headings = append(headings, heading{level: level, text: text, lineStart: offset, bodyStart: end}) + } + } + offset = end + } + + sections := make([]*sourceNode, 0, len(headings)) + for index, current := range headings { + bodyEnd := len(data) + for next := index + 1; next < len(headings); next++ { + if headings[next].level <= current.level { + bodyEnd = headings[next].lineStart + break + } + } + body := markdownTextNode(data[current.bodyStart:bodyEnd], current.bodyStart, bodyEnd) + sections = append(sections, &sourceNode{fields: map[string]*sourceNode{ + "heading": {value: current.text}, + "body": body, + }}) + } + return sections +} + +func markdownFence(line string) (byte, int) { + if line == "" || line[0] != '`' && line[0] != '~' { + return 0, 0 + } + length := 1 + for length < len(line) && line[length] == line[0] { + length++ + } + return line[0], length +} + +func markdownHeadingText(text string) string { + text = strings.TrimSpace(text) + lastSpace := strings.LastIndexByte(text, ' ') + if lastSpace < 0 || lastSpace == len(text)-1 { + return text + } + for _, character := range text[lastSpace+1:] { + if character != '#' { + return text + } + } + return strings.TrimSpace(text[:lastSpace]) +} + +func markdownTextNode(data []byte, start, end int) *sourceNode { + value := normalizeMarkdown(string(data)) + lineEnding := "\n" + if bytes.Contains(data, []byte("\r\n")) { + lineEnding = "\r\n" + } + contentStart, contentEnd := 0, len(data) + for contentStart < len(data) && isMarkdownSpace(data[contentStart]) { + contentStart++ + } + for contentEnd > contentStart && isMarkdownSpace(data[contentEnd-1]) { + contentEnd-- + } + prefix := bytes.Clone(data[:contentStart]) + suffix := bytes.Clone(data[contentEnd:]) + return stringNode(value, start, end, func(text string) ([]byte, error) { + text = normalizeMarkdown(text) + text = strings.ReplaceAll(text, "\n", lineEnding) + result := append(bytes.Clone(prefix), []byte(text)...) + return append(result, suffix...), nil + }) +} + +func isMarkdownSpace(value byte) bool { + return value == ' ' || value == '\t' || value == '\r' || value == '\n' +} + +func parseAgentSkill(document Document) (*sourceNode, error) { + if !utf8.Valid(document.Data) { + return nil, fmt.Errorf("%s: source is not valid UTF-8", document.Path) + } + frontMatter, bodyStart, err := splitFrontMatter(document.Data) + if err != nil { + return nil, fmt.Errorf("%s: %w", document.Path, err) + } + var yamlNode yaml.Node + decoder := yaml.NewDecoder(bytes.NewReader(frontMatter.data)) + if err := decoder.Decode(&yamlNode); err != nil { + return nil, fmt.Errorf("%s: invalid front matter: %w", document.Path, err) + } + description, err := yamlScalarNode(frontMatter, &yamlNode, "description") + if err != nil { + return nil, fmt.Errorf("%s: %w", document.Path, err) + } + body := document.Data[bodyStart:] + if len(body) == 0 || strings.TrimSpace(string(body)) == "" { + return nil, fmt.Errorf("%s: skill Markdown is required", document.Path) + } + bodyNode := exactMarkdownNode(body, bodyStart, len(document.Data)) + return &sourceNode{ + fields: map[string]*sourceNode{ + "frontmatter": {fields: map[string]*sourceNode{"description": description}}, + "body": bodyNode, + }, + }, nil +} + +type frontMatterRegion struct { + data []byte + startOffset int +} + +func splitFrontMatter(data []byte) (frontMatterRegion, int, error) { + offset := 0 + if bytes.HasPrefix(data, []byte("\ufeff")) { + offset = len([]byte("\ufeff")) + } + firstEnd := nextLineEnd(data, offset) + if strings.TrimSpace(string(data[offset:firstEnd])) != "---" { + return frontMatterRegion{}, 0, fmt.Errorf("missing YAML front matter") + } + contentStart := firstEnd + for lineStart := firstEnd; lineStart < len(data); { + lineEnd := nextLineEnd(data, lineStart) + if strings.TrimSpace(string(data[lineStart:lineEnd])) == "---" { + bodyStart := lineEnd + if bytes.HasPrefix(data[bodyStart:], []byte("\r\n")) { + bodyStart += 2 + } else if bytes.HasPrefix(data[bodyStart:], []byte("\n")) { + bodyStart++ + } + return frontMatterRegion{data: data[contentStart:lineStart], startOffset: contentStart}, bodyStart, nil + } + lineStart = lineEnd + } + return frontMatterRegion{}, 0, fmt.Errorf("unclosed YAML front matter") +} + +func nextLineEnd(data []byte, start int) int { + if index := bytes.IndexByte(data[start:], '\n'); index >= 0 { + return start + index + 1 + } + return len(data) +} + +func yamlScalarNode(region frontMatterRegion, document *yaml.Node, key string) (*sourceNode, error) { + if len(document.Content) == 0 || document.Content[0].Kind != yaml.MappingNode { + return nil, fmt.Errorf("front matter must be a mapping") + } + mapping := document.Content[0] + for index := 0; index+1 < len(mapping.Content); index += 2 { + name, value := mapping.Content[index], mapping.Content[index+1] + if name.Value != key { + continue + } + if value.Kind != yaml.ScalarNode || value.Tag != "!!str" { + return nil, fmt.Errorf("front matter %s must be a string", key) + } + start, end, raw, err := yamlScalarSpan(region, value) + if err != nil { + return nil, err + } + if value.Style == yaml.LiteralStyle || value.Style == yaml.FoldedStyle { + return &sourceNode{ + value: value.Value, start: start, end: end, capability: CapabilityReadOnly, + reason: "YAML block scalars are read-only", + }, nil + } + return stringNode(value.Value, start, end, func(text string) ([]byte, error) { + return encodeYAMLString(text, raw) + }), nil + } + return nil, syncdomain.NewProblem( + syncdomain.ErrorPathNotFound, + fmt.Sprintf("front matter field %q was not found", key), + "Add the field or update the map path.", + ) +} + +func yamlScalarSpan(region frontMatterRegion, node *yaml.Node) (int, int, string, error) { + lines := bytes.SplitAfter(region.data, []byte("\n")) + if node.Line < 1 || node.Line > len(lines) || node.Column < 1 { + return 0, 0, "", fmt.Errorf("cannot locate YAML scalar") + } + start := region.startOffset + for index := 0; index < node.Line-1; index++ { + start += len(lines[index]) + } + line := lines[node.Line-1] + line = bytes.TrimRight(line, "\r\n") + localStart := byteOffsetForColumn(line, node.Column) + start += localStart + localEnd := yamlTokenEnd(line, localStart) + if localStart > localEnd { + return 0, 0, "", fmt.Errorf("cannot locate YAML scalar") + } + raw := string(line[localStart:localEnd]) + return start, start + len(raw), raw, nil +} + +func byteOffsetForColumn(line []byte, column int) int { + offset := 0 + for current := 1; current < column && offset < len(line); current++ { + _, width := utf8.DecodeRune(line[offset:]) + offset += width + } + return offset +} + +func yamlTokenEnd(line []byte, start int) int { + if start >= len(line) { + return start + } + quote := line[start] + if quote == '\'' || quote == '"' { + escaped := false + for index := start + 1; index < len(line); index++ { + if quote == '"' && line[index] == '\\' && !escaped { + escaped = true + continue + } + if line[index] == quote && !escaped { + if quote == '\'' && index+1 < len(line) && line[index+1] == '\'' { + index++ + continue + } + return index + 1 + } + escaped = false + } + return len(line) + } + end := len(line) + if comment := bytes.Index(line[start:], []byte(" #")); comment >= 0 { + end = start + comment + } + for _, delimiter := range []byte{',', ']', '}'} { + if index := bytes.IndexByte(line[start:end], delimiter); index >= 0 { + end = start + index + } + } + for end > start && (line[end-1] == ' ' || line[end-1] == '\t') { + end-- + } + return end +} + +func quoteYAML(value, previous string) string { + if strings.HasPrefix(previous, "'") { + if strings.ContainsAny(value, "\r\n") { + encoded, _ := json.Marshal(value) + return string(encoded) + } + return "'" + strings.ReplaceAll(value, "'", "''") + "'" + } + if strings.HasPrefix(previous, `"`) { + encoded, _ := json.Marshal(value) + return string(encoded) + } + if value != "" && !strings.ContainsAny(value, ":\n\r#{}[],&*!|>'\"%@`") && + value == strings.TrimSpace(value) { + var decoded any + if yaml.Unmarshal([]byte(value), &decoded) == nil { + if _, isString := decoded.(string); isString { + return value + } + } + } + encoded, _ := json.Marshal(value) + return string(encoded) +} + +func encodeYAMLString(value, previous string) ([]byte, error) { + for _, character := range value { + if unicode.IsControl(character) && character != '\n' && character != '\r' && character != '\t' { + return nil, fmt.Errorf("YAML string contains unsupported control character U+%04X", character) + } + } + return []byte(quoteYAML(value, previous)), nil +} + +func exactMarkdownNode(data []byte, start, end int) *sourceNode { + value := string(data) + finalNewline := bytes.HasSuffix(data, []byte("\n")) + return stringNode(value, start, end, func(text string) ([]byte, error) { + if finalNewline && text != "" && !strings.HasSuffix(text, "\n") { + text += "\n" + } + return []byte(text), nil + }) +} diff --git a/internal/sync/reference/fixture_test.go b/internal/sync/reference/fixture_test.go new file mode 100644 index 000000000..98d6c46e7 --- /dev/null +++ b/internal/sync/reference/fixture_test.go @@ -0,0 +1,25 @@ +package reference + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func fixtureFile(t *testing.T, name string) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", filepath.FromSlash(name))) + require.NoError(t, err) + return data +} + +func materializeFixture(t *testing.T, fixture, realPath string) (string, []byte) { + t.Helper() + data := fixtureFile(t, fixture) + path := filepath.Join(t.TempDir(), filepath.FromSlash(realPath)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, data, 0o644)) + return path, data +} diff --git a/internal/sync/reference/path.go b/internal/sync/reference/path.go new file mode 100644 index 000000000..5998cbc10 --- /dev/null +++ b/internal/sync/reference/path.go @@ -0,0 +1,270 @@ +package reference + +import ( + "encoding/json" + "fmt" + "strconv" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type sourcePath struct { + root bool + steps []pathStep +} + +type pathStep struct { + key string + filters []pathFilter + index *int +} + +type pathFilter struct { + key string + value string +} + +// ParsePath checks the shared select and map path grammar. +func ParsePath(text string) error { + _, err := parsePath(text) + return err +} + +func parsePath(text string) (sourcePath, error) { + if text == "$" { + return sourcePath{root: true}, nil + } + if text == "" { + return sourcePath{}, pathSyntaxError(text, 0, "path is empty") + } + + parser := pathParser{text: text} + key, err := parser.readKey() + if err != nil { + return sourcePath{}, err + } + result := sourcePath{steps: []pathStep{{key: key}}} + for parser.offset < len(text) { + switch text[parser.offset] { + case '.': + parser.offset++ + key, err = parser.readIdentifier() + if err != nil { + return sourcePath{}, err + } + result.steps = append(result.steps, pathStep{key: key}) + case '[': + step, kind, err := parser.readBracket() + if err != nil { + return sourcePath{}, err + } + if kind == bracketFilter { + last := len(result.steps) - 1 + if result.steps[last].index != nil { + return sourcePath{}, pathSyntaxError(text, parser.offset, "a selector must follow a field") + } + if len(result.steps[last].filters) != 0 { + return sourcePath{}, pathSyntaxError(text, parser.offset, "a path step can have only one selector") + } + result.steps[last].filters = step.filters + } else { + result.steps = append(result.steps, step) + } + default: + return sourcePath{}, pathSyntaxError(text, parser.offset, "expected a dot or bracket") + } + } + return result, nil +} + +type pathParser struct { + text string + offset int +} + +type bracketKind int + +const ( + bracketField bracketKind = iota + bracketFilter + bracketIndex +) + +func (parser *pathParser) readKey() (string, error) { + if parser.offset < len(parser.text) && parser.text[parser.offset] == '[' { + step, kind, err := parser.readBracket() + if err != nil { + return "", err + } + if kind != bracketField { + return "", pathSyntaxError(parser.text, parser.offset, "a path cannot start with a selector or index") + } + return step.key, nil + } + return parser.readIdentifier() +} + +func (parser *pathParser) readIdentifier() (string, error) { + start := parser.offset + if start >= len(parser.text) || !identifierStart(parser.text[start]) { + return "", pathSyntaxError(parser.text, start, "expected an identifier") + } + parser.offset++ + for parser.offset < len(parser.text) && identifierContinue(parser.text[parser.offset]) { + parser.offset++ + } + return parser.text[start:parser.offset], nil +} + +func (parser *pathParser) readBracket() (pathStep, bracketKind, error) { + start := parser.offset + parser.offset++ + if parser.offset < len(parser.text) && parser.text[parser.offset] >= '0' && parser.text[parser.offset] <= '9' { + return parser.readIndex(start) + } + key, err := parser.readBracketKey() + if err != nil { + return pathStep{}, bracketField, err + } + if parser.offset >= len(parser.text) { + return pathStep{}, bracketField, pathSyntaxError(parser.text, start, "unclosed bracket") + } + if parser.text[parser.offset] == ']' { + parser.offset++ + return pathStep{key: key}, bracketField, nil + } + if parser.text[parser.offset] != '=' { + return pathStep{}, bracketField, pathSyntaxError(parser.text, parser.offset, "expected equals or closing bracket") + } + parser.offset++ + if parser.offset >= len(parser.text) || parser.text[parser.offset] != '"' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "selector value must be a JSON string") + } + value, err := parser.readJSONString() + if err != nil { + return pathStep{}, bracketFilter, err + } + filters := []pathFilter{{key: key, value: value}} + for parser.offset < len(parser.text) && parser.text[parser.offset] == ',' { + parser.offset++ + nextKey, err := parser.readBracketKey() + if err != nil { + return pathStep{}, bracketFilter, err + } + if parser.offset >= len(parser.text) || parser.text[parser.offset] != '=' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "expected equals") + } + parser.offset++ + if parser.offset >= len(parser.text) || parser.text[parser.offset] != '"' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "selector value must be a JSON string") + } + nextValue, err := parser.readJSONString() + if err != nil { + return pathStep{}, bracketFilter, err + } + filters = append(filters, pathFilter{key: nextKey, value: nextValue}) + } + if parser.offset >= len(parser.text) || parser.text[parser.offset] != ']' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "expected a closing bracket") + } + parser.offset++ + return pathStep{filters: filters}, bracketFilter, nil +} + +func (parser *pathParser) readIndex(start int) (pathStep, bracketKind, error) { + numberStart := parser.offset + for parser.offset < len(parser.text) && parser.text[parser.offset] >= '0' && parser.text[parser.offset] <= '9' { + parser.offset++ + } + raw := parser.text[numberStart:parser.offset] + if len(raw) > 1 && raw[0] == '0' { + return pathStep{}, bracketIndex, pathSyntaxError(parser.text, numberStart, "an index cannot have a leading zero") + } + if parser.offset >= len(parser.text) || parser.text[parser.offset] != ']' { + return pathStep{}, bracketIndex, pathSyntaxError(parser.text, parser.offset, "an index must be a decimal integer") + } + value, err := strconv.ParseUint(raw, 10, 31) + if err != nil { + return pathStep{}, bracketIndex, pathSyntaxError(parser.text, numberStart, "index exceeds 2147483647") + } + parser.offset++ + index := int(value) + return pathStep{index: &index}, bracketIndex, nil +} + +func (parser *pathParser) readBracketKey() (string, error) { + if parser.offset < len(parser.text) && parser.text[parser.offset] == '"' { + return parser.readJSONString() + } + return parser.readIdentifier() +} + +func (parser *pathParser) readJSONString() (string, error) { + start := parser.offset + parser.offset++ + for parser.offset < len(parser.text) { + switch parser.text[parser.offset] { + case '\\': + parser.offset += 2 + case '"': + parser.offset++ + quoted := parser.text[start:parser.offset] + if err := rejectUnpairedSurrogate(quoted); err != nil { + return "", pathSyntaxError(parser.text, start, err.Error()) + } + var value string + if err := json.Unmarshal([]byte(quoted), &value); err != nil { + return "", pathSyntaxError(parser.text, start, "invalid JSON string") + } + return value, nil + default: + parser.offset++ + } + } + return "", pathSyntaxError(parser.text, start, "unclosed JSON string") +} + +func rejectUnpairedSurrogate(quoted string) error { + for index := 0; index+5 < len(quoted); index++ { + if quoted[index:index+2] != `\u` { + continue + } + code, err := strconv.ParseUint(quoted[index+2:index+6], 16, 16) + if err != nil { + continue + } + switch { + case code >= 0xd800 && code <= 0xdbff: + if index+11 >= len(quoted) || quoted[index+6:index+8] != `\u` { + return fmt.Errorf("unpaired Unicode surrogate") + } + low, err := strconv.ParseUint(quoted[index+8:index+12], 16, 16) + if err != nil || low < 0xdc00 || low > 0xdfff { + return fmt.Errorf("unpaired Unicode surrogate") + } + index += 6 + case code >= 0xdc00 && code <= 0xdfff: + return fmt.Errorf("unpaired Unicode surrogate") + } + } + return nil +} + +func pathSyntaxError(path string, offset int, reason string) error { + if offset > len(path) { + offset = len(path) + } + return syncdomain.NewProblem( + syncdomain.ErrorInvalidPathSyntax, + fmt.Sprintf("invalid path at byte %d: %s", offset, reason), + "Use $, named fields, zero-based indexes, keyed selectors, or quoted literal keys.", + ) +} + +func identifierStart(value byte) bool { + return value == '_' || value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z' +} + +func identifierContinue(value byte) bool { + return identifierStart(value) || value >= '0' && value <= '9' +} diff --git a/internal/sync/reference/registry.go b/internal/sync/reference/registry.go new file mode 100644 index 000000000..8181b6127 --- /dev/null +++ b/internal/sync/reference/registry.go @@ -0,0 +1,119 @@ +package reference + +import ( + "fmt" + "sync" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// Adapter inspects one source format and supplies its conventional mapping. +// It receives bytes only and never reads files, runs code, or uses the network. +type Adapter interface { + Format() string + Inspect(SourceBundle) (Inspection, error) + Defaults(syncdomain.Kind) (selectPath string, fieldMap map[string]string) +} + +// DependencyReporter lists repository files that can contribute mapped values. +type DependencyReporter interface { + Dependencies(Document) ([]string, error) +} + +// Inspection is the logical document that an adapter exposes to select paths. +type Inspection struct { + root *sourceNode +} + +// Node is one logical value exposed by an adapter. +type Node struct { + Value any + Fields map[string]*Node + Items []*Node + Start int + End int + File string + Capability syncdomain.Capability + Reason string + Code syncdomain.ErrorCode + Encode func(any) ([]byte, error) +} + +// NewInspection creates an inspection from an adapter's logical root. +func NewInspection(root *Node) Inspection { + return Inspection{root: importNode(root)} +} + +func importNode(node *Node) *sourceNode { + if node == nil { + return nil + } + result := &sourceNode{ + value: node.Value, start: node.Start, end: node.End, file: node.File, + capability: node.Capability, reason: node.Reason, code: node.Code, encode: node.Encode, + } + if len(node.Fields) != 0 { + result.fields = make(map[string]*sourceNode, len(node.Fields)) + for key, child := range node.Fields { + result.fields[key] = importNode(child) + } + } + for _, child := range node.Items { + result.items = append(result.items, importNode(child)) + } + return result +} + +type adapterRegistry struct { + mu sync.RWMutex + adapters map[string]Adapter +} + +var formats = adapterRegistry{adapters: make(map[string]Adapter)} + +// Register adds one independent source format. +func Register(adapter Adapter) error { + if adapter == nil || adapter.Format() == "" { + return fmt.Errorf("adapter format is required") + } + formats.mu.Lock() + defer formats.mu.Unlock() + if _, exists := formats.adapters[adapter.Format()]; exists { + return fmt.Errorf("source format %q is already registered", adapter.Format()) + } + formats.adapters[adapter.Format()] = adapter + return nil +} + +func registeredAdapter(format string) (Adapter, error) { + formats.mu.RLock() + adapter := formats.adapters[format] + formats.mu.RUnlock() + if adapter == nil { + return nil, syncdomain.NewProblem( + syncdomain.ErrorUnsupportedFormat, + fmt.Sprintf("source format %q is not supported", format), + "Use a registered format name.", + ) + } + return adapter, nil +} + +// DependencyPaths returns repository dependencies reported by one format. +func DependencyPaths(format string, primary Document) ([]string, error) { + adapter, err := registeredAdapter(format) + if err != nil { + return nil, err + } + reporter, ok := adapter.(DependencyReporter) + if !ok { + return nil, nil + } + return reporter.Dependencies(primary) +} + +func mustRegister(adapter Adapter) { + if err := Register(adapter); err != nil { + panic(err) + } +} diff --git a/internal/sync/reference/registry_external_test.go b/internal/sync/reference/registry_external_test.go new file mode 100644 index 000000000..72e37cc7c --- /dev/null +++ b/internal/sync/reference/registry_external_test.go @@ -0,0 +1,42 @@ +package reference_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/reference" +) + +type externalFakeAdapter struct{} + +func (externalFakeAdapter) Format() string { return "external-fake" } +func (externalFakeAdapter) Inspect(bundle reference.SourceBundle) (reference.Inspection, error) { + document := bundle.Primary + return reference.NewInspection(&reference.Node{ + Value: string(document.Data), Start: 0, End: len(document.Data), + Capability: syncdomain.CapabilityDirect, + Encode: func(value any) ([]byte, error) { return []byte(value.(string)), nil }, + }), nil +} +func (externalFakeAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { + return "$", map[string]string{"prompt": "$"} +} + +func TestExternalAdapterNeedsOnlyRegistration(t *testing.T) { + require.NoError(t, reference.Register(externalFakeAdapter{})) + id := syncdomain.VariationID("project", "config", "default") + + projection, err := reference.ParseSource( + reference.SourceBundle{ + Primary: reference.Document{Path: "prompt.fake", Data: []byte("External adapter")}, + }, + id, + syncdomain.VariationModeAgent, + syncdomain.Reference{File: "prompt.fake", Format: "external-fake"}, + ) + + require.NoError(t, err) + require.Equal(t, "External adapter", *projection.(syncdomain.VariationProjection).Instructions) +} diff --git a/internal/sync/reference/repository_paths_test.go b/internal/sync/reference/repository_paths_test.go new file mode 100644 index 000000000..72265e8cc --- /dev/null +++ b/internal/sync/reference/repository_paths_test.go @@ -0,0 +1,61 @@ +package reference_test + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAdapterFixturesHaveNoToolLoadedInstructionPaths(t *testing.T) { + root := filepath.Join(repositoryRoot(t), "internal", "sync") + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + require.NoError(t, err) + relative, err := filepath.Rel(root, path) + require.NoError(t, err) + relative = filepath.ToSlash(relative) + if !insideTestdata(relative) { + return nil + } + require.Falsef(t, forbiddenInstructionPath(relative), "forbidden adapter fixture path: %s", relative) + return nil + }) + require.NoError(t, err) +} + +func repositoryRoot(t *testing.T) string { + t.Helper() + directory, err := os.Getwd() + require.NoError(t, err) + for { + if _, err := os.Stat(filepath.Join(directory, "go.mod")); err == nil { + return directory + } + parent := filepath.Dir(directory) + require.NotEqual(t, directory, parent, "repository root was not found") + directory = parent + } +} + +func forbiddenInstructionPath(path string) bool { + parts := strings.Split(path, "/") + for _, part := range parts { + switch part { + case ".cursor", ".claude", ".agents", ".cursorrules", "AGENTS.md", "CLAUDE.md", "SKILL.md": + return true + } + } + return path == ".github/copilot-instructions.md" || strings.HasPrefix(path, ".github/instructions/") +} + +func insideTestdata(path string) bool { + for _, part := range strings.Split(path, "/") { + if part == "testdata" { + return true + } + } + return false +} diff --git a/internal/sync/reference/source.go b/internal/sync/reference/source.go new file mode 100644 index 000000000..57faa027c --- /dev/null +++ b/internal/sync/reference/source.go @@ -0,0 +1,1027 @@ +package reference + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "reflect" + "slices" + "strings" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// Capability is the shared mapped-field capability. +type Capability = syncdomain.Capability + +const ( + CapabilityDirect = syncdomain.CapabilityDirect + CapabilityPatch = syncdomain.CapabilityPatch + CapabilityReadOnly = syncdomain.CapabilityReadOnly + CapabilityReject = syncdomain.CapabilityReject +) + +// Document is one repository file supplied to an adapter. +type Document struct { + Path string + Data []byte +} + +// SourceBundle contains a primary source and optional repository dependencies. +type SourceBundle struct { + Primary Document + Dependencies map[string]Document +} + +// TextEdit replaces one exact byte range. +type TextEdit struct { + File string + Start int + End int + Expected []byte + Replacement []byte +} + +// FieldUpdate is the update capability and edits for one mapped field. +type FieldUpdate struct { + Capability Capability + Edits []TextEdit + Reason string +} + +// UpdateResult contains one result for each changed field. +type UpdateResult struct { + Fields map[syncdomain.Field]FieldUpdate +} + +// NormalizeReference validates a descriptor and fills conventional defaults. +func NormalizeReference(id syncdomain.ResourceID, reference syncdomain.Reference) (syncdomain.Reference, error) { + if reference.File == "" { + return syncdomain.Reference{}, invalidReference(id, "file is required", "Pass --file.") + } + adapter, err := registeredAdapter(reference.Format) + if err != nil { + return syncdomain.Reference{}, err + } + defaultSelect, defaultMap := adapter.Defaults(id.Kind) + if reference.Select == "" { + if defaultSelect == "" { + return syncdomain.Reference{}, invalidReference( + id, fmt.Sprintf("select is required for format %q", reference.Format), "Pass --select.", + ) + } + reference.Select = defaultSelect + } + if err := ParsePath(reference.Select); err != nil { + return syncdomain.Reference{}, annotateProblem( + fmt.Errorf("select %q: %w", reference.Select, err), id, reference.File, + ) + } + if len(reference.Map) == 0 { + if len(defaultMap) == 0 { + return syncdomain.Reference{}, invalidReference( + id, "map is required", "Pass at least one --map field=path value.", + ) + } + reference.Map = defaultMap + } + for target := range reference.Map { + if err := ParsePath(target); err != nil { + return syncdomain.Reference{}, annotateProblem( + fmt.Errorf("map target %q: %w", target, err), id, reference.File, + ) + } + } + fields, err := syncdomain.ProjectionFields(reference) + if err != nil { + return syncdomain.Reference{}, invalidReference(id, err.Error(), "Use a field valid for this resource kind.") + } + wholeParameters := false + parameterKeys := map[string]syncdomain.Field{} + for _, field := range fields { + if field == syncdomain.FieldTools || field == syncdomain.FieldSkills { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("map key %q is not allowed", field), + fmt.Sprintf("List %s keys under top-level %q.", field, field), + ) + } + if !fieldAllowed(id.Kind, field) { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("field %q is not valid for %s", field, id.Kind), + "Use a mapped field supported by this resource kind.", + ) + } + if field == syncdomain.FieldModelParameters { + wholeParameters = true + } + if key, parameter := syncdomain.ModelParameterKey(field); parameter { + if key == "tools" { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("map target %q duplicates the tools field", field), + "List tool keys under top-level \"tools\".", + ) + } + if previous, duplicate := parameterKeys[key]; duplicate { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("map targets %q and %q select the same model parameter", previous, field), + "Map each model parameter key once.", + ) + } + parameterKeys[key] = field + } + if err := ParsePath(reference.Map[string(field)]); err != nil { + return syncdomain.Reference{}, annotateProblem( + fmt.Errorf("map field %q: %w", field, err), id, reference.File, + ) + } + } + if wholeParameters && len(parameterKeys) != 0 { + return syncdomain.Reference{}, invalidReference( + id, + "map target \"model.parameters\" overlaps an individual model parameter target", + "Map the full parameters object or individual parameter keys.", + ) + } + return reference, nil +} + +func invalidReference(id syncdomain.ResourceID, message, suggestion string) error { + return syncdomain.NewProblem( + syncdomain.ErrorInvalidReference, message, suggestion, + ).WithResource(id) +} + +// ParseSource reads one mapped resource from current source bytes. +func ParseSource( + bundle SourceBundle, + id syncdomain.ResourceID, + mode syncdomain.VariationMode, + reference syncdomain.Reference, +) (syncdomain.Projection, error) { + projection, _, err := ParseSourceWithSelections(bundle, id, mode, reference) + return projection, err +} + +// ParseSourceWithSelections reads one mapped resource and its indexed selections. +func ParseSourceWithSelections( + bundle SourceBundle, + id syncdomain.ResourceID, + mode syncdomain.VariationMode, + reference syncdomain.Reference, +) (syncdomain.Projection, []syncdomain.IndexedSelection, error) { + reference, err := NormalizeReference(id, reference) + if err != nil { + return nil, nil, err + } + root, err := parseSourceBundle(bundle, reference.Format) + if err != nil { + return nil, nil, annotateProblem(err, id, bundle.Primary.Path) + } + selected, selectedIndexes, err := resolveSourcePathTracked(root, reference.Select, "select") + if err != nil { + return nil, nil, annotateProblem(fmt.Errorf("select %q: %w", reference.Select, err), id, bundle.Primary.Path) + } + selections, err := indexedSelections(selectedIndexes, selected) + if err != nil { + return nil, nil, annotateProblem(err, id, bundle.Primary.Path) + } + fields := make([]string, 0, len(reference.Map)) + for field := range reference.Map { + fields = append(fields, field) + } + slices.Sort(fields) + for _, field := range fields { + _, indexes, err := resolveSourcePathTracked(selected, reference.Map[field], "map."+field) + if err != nil { + return nil, nil, annotateProblem(fmt.Errorf("map field %q: %w", field, err), id, bundle.Primary.Path) + } + mappedSelections, err := indexedSelections(indexes, selected) + if err != nil { + return nil, nil, annotateProblem(err, id, bundle.Primary.Path) + } + selections = append(selections, mappedSelections...) + } + slices.SortFunc(selections, func(left, right syncdomain.IndexedSelection) int { + return strings.Compare(left.Path, right.Path) + }) + projection, err := buildProjection(id.Kind, mode, selected, reference.Map) + if variation, ok := projection.(syncdomain.VariationProjection); ok { + if reference.Tools != nil { + keys := attachmentKeys(*reference.Tools) + variation.Tools = &keys + variation.FieldStates[syncdomain.FieldTools] = syncdomain.FieldState{ + Capability: syncdomain.CapabilityDirect, Known: true, + } + } + if reference.Skills != nil { + keys := attachmentKeys(*reference.Skills) + variation.Skills = &keys + variation.FieldStates[syncdomain.FieldSkills] = syncdomain.FieldState{ + Capability: syncdomain.CapabilityDirect, Known: true, + } + } + projection = variation + } + return projection, selections, annotateProblem(err, id, bundle.Primary.Path) +} + +// UpdateSource plans exact edits for changed mapped fields. +func UpdateSource( + bundle SourceBundle, + id syncdomain.ResourceID, + mode syncdomain.VariationMode, + reference syncdomain.Reference, + desired syncdomain.Projection, +) (UpdateResult, error) { + reference, err := NormalizeReference(id, reference) + if err != nil { + return UpdateResult{}, err + } + if desired.Kind() != id.Kind { + return UpdateResult{}, fmt.Errorf("projection kind %q does not match %s", desired.Kind(), id.Label()) + } + root, err := parseSourceBundle(bundle, reference.Format) + if err != nil { + return UpdateResult{}, annotateProblem(err, id, bundle.Primary.Path) + } + selected, err := resolveSourcePath(root, reference.Select) + if err != nil { + return UpdateResult{}, annotateProblem(fmt.Errorf("select %q: %w", reference.Select, err), id, bundle.Primary.Path) + } + + result := UpdateResult{Fields: make(map[syncdomain.Field]FieldUpdate)} + for name, sourcePath := range reference.Map { + field := syncdomain.Field(name) + wanted, present, err := projectionField(desired, field) + if err != nil { + return UpdateResult{}, err + } + if !present { + continue + } + current, err := resolveSourcePath(selected, sourcePath) + if err != nil { + return UpdateResult{}, fmt.Errorf("%s: map field %q: %w", bundle.Primary.Path, field, err) + } + if field == syncdomain.FieldPrompt || field == syncdomain.FieldMessages { + if messages, ok := wanted.([]syncdomain.Message); ok { + if currentMessages, sourceIsMessages := current.value.([]syncdomain.Message); sourceIsMessages { + update := updateMessageContents(bundle, current, currentMessages, messages) + if len(update.Edits) != 0 || update.Capability != CapabilityDirect { + result.Fields[field] = update + } + continue + } + if _, sourceIsText := current.value.(string); sourceIsText { + if len(messages) != 1 || messages[0].Role != syncdomain.RoleSystem { + result.Fields[field] = FieldUpdate{ + Capability: CapabilityReadOnly, + Reason: "text sources support one system message", + } + continue + } + wanted = messages[0].Content + } + } + } + if semanticEqual(current.value, wanted) { + continue + } + update := FieldUpdate{Capability: current.capability, Reason: current.reason} + if current.capability == CapabilityDirect || current.capability == CapabilityPatch { + replacement, err := current.encode(wanted) + if err != nil { + update.Capability, update.Reason = CapabilityReadOnly, err.Error() + } else { + document, ok := bundleDocument(bundle, current.file) + if !ok { + update.Capability, update.Reason = CapabilityReadOnly, "source dependency is not available" + result.Fields[field] = update + continue + } + update.Edits = []TextEdit{{ + File: document.Path, + Start: current.start, + End: current.end, + Expected: bytes.Clone(document.Data[current.start:current.end]), + Replacement: replacement, + }} + } + } + result.Fields[field] = update + } + return result, nil +} + +func updateMessageContents( + bundle SourceBundle, + current *sourceNode, + have, want []syncdomain.Message, +) FieldUpdate { + if len(have) != len(want) { + return FieldUpdate{ + Capability: CapabilityReadOnly, + Reason: "add or remove messages by hand", + } + } + var edits []TextEdit + for index := range have { + if have[index].Role != want[index].Role { + return FieldUpdate{ + Capability: CapabilityReadOnly, + Reason: "reorder messages or change roles by hand", + } + } + if have[index].Content == want[index].Content { + continue + } + if index >= len(current.items) || current.items[index].fields["content"] == nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: "message content is not a writable literal"} + } + content := current.items[index].fields["content"] + if content.capability != CapabilityDirect || content.encode == nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: content.reason} + } + replacement, err := content.encode(want[index].Content) + if err != nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: err.Error()} + } + document, ok := bundleDocument(bundle, content.file) + if !ok { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: "source dependency is not available"} + } + edits = append(edits, TextEdit{ + File: document.Path, Start: content.start, End: content.end, + Expected: bytes.Clone(document.Data[content.start:content.end]), Replacement: replacement, + }) + } + return FieldUpdate{Capability: CapabilityDirect, Edits: edits} +} + +func bundleDocument(bundle SourceBundle, file string) (Document, bool) { + if file == "" || file == bundle.Primary.Path { + return bundle.Primary, true + } + document, ok := bundle.Dependencies[file] + return document, ok +} + +func annotateProblem(err error, id syncdomain.ResourceID, file string) error { + if err == nil { + return nil + } + if problem, ok := syncdomain.ProblemFrom(err); ok { + _ = problem.WithResource(id).WithFile(file, 0, 0) + } + return err +} + +type sourceNode struct { + value any + fields map[string]*sourceNode + items []*sourceNode + collection bool + start int + end int + file string + capability Capability + reason string + code syncdomain.ErrorCode + encode func(any) ([]byte, error) +} + +func resolveSourcePath(root *sourceNode, text string) (*sourceNode, error) { + node, _, err := resolveSourcePathTracked(root, text, "") + return node, err +} + +type indexedResolution struct { + path string + collectionLength int + item *sourceNode +} + +func resolveSourcePathTracked(root *sourceNode, text, scope string) (*sourceNode, []indexedResolution, error) { + path, err := parsePath(text) + if err != nil { + return nil, nil, err + } + if path.root { + return root, nil, nil + } + current := root + var indexes []indexedResolution + for _, step := range path.steps { + if step.index != nil { + if !sourceCollection(current) { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathTypeMismatch, + fmt.Sprintf("index %d needs a collection", *step.index), + "Select a collection before this index.", + ) + } + if *step.index >= len(current.items) { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorIndexOutOfRange, + fmt.Sprintf("index %d is out of range for collection of length %d", *step.index, len(current.items)), + "Use an index within the collection.", + ) + } + collectionLength := len(current.items) + current = current.items[*step.index] + indexes = append(indexes, indexedResolution{ + path: fmt.Sprintf("%s#%d", scope, len(indexes)), + collectionLength: collectionLength, + item: current, + }) + continue + } + if current.fields == nil { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathTypeMismatch, + fmt.Sprintf("%q does not contain fields", step.key), + "Select an object before this field.", + ) + } + next, ok := current.fields[step.key] + if !ok { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathNotFound, + fmt.Sprintf("field %q was not found", step.key), + "Update the select or map path.", + ) + } + current = next + if len(step.filters) == 0 { + continue + } + var matches []*sourceNode + for _, item := range current.items { + matched := true + for _, filter := range step.filters { + identity, ok := item.fields[filter.key] + if !ok { + matched = false + break + } + text, ok := identity.value.(string) + if !ok { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorFilterTypeMismatch, + fmt.Sprintf("selector field %q is not a string", filter.key), + "Select a direct string identity field.", + ) + } + if text != filter.value { + matched = false + break + } + } + if matched { + matches = append(matches, item) + } + } + selector := formatFilters(step.filters) + switch len(matches) { + case 0: + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathNotFound, + fmt.Sprintf("selector %s matched no values", selector), + "Update the selector or restore the source value.", + ) + case 1: + current = matches[0] + default: + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathAmbiguous, + fmt.Sprintf("selector %s matched %d values", selector, len(matches)), + "Use a selector that matches one resource.", + ) + } + } + return current, indexes, nil +} + +func sourceCollection(node *sourceNode) bool { + if node == nil { + return false + } + if node.collection || node.items != nil { + return true + } + value := reflect.ValueOf(node.value) + return value.IsValid() && (value.Kind() == reflect.Array || value.Kind() == reflect.Slice) +} + +func indexedSelections(indexes []indexedResolution, selectedRoot *sourceNode) ([]syncdomain.IndexedSelection, error) { + selections := make([]syncdomain.IndexedSelection, 0, len(indexes)) + var ancestors []string + for _, index := range indexes { + identity := directSelectionIdentity(index.item) + if identity == nil { + // Without scalar identity, bind the nearest selected resource root and prior indexed ancestors. + identity = struct { + Root any `json:"root"` + Ancestors []string `json:"ancestors,omitempty"` + Item any `json:"item"` + }{ + Root: stableNodeValue(selectedRoot, map[*sourceNode]bool{}), + Ancestors: slices.Clone(ancestors), + Item: stableNodeValue(index.item, map[*sourceNode]bool{}), + } + } + fingerprint, err := selectionFingerprint(identity) + if err != nil { + return nil, err + } + selections = append(selections, syncdomain.IndexedSelection{ + Path: index.path, CollectionLength: index.collectionLength, Fingerprint: fingerprint, + }) + ancestors = append(ancestors, fingerprint) + } + return selections, nil +} + +func directSelectionIdentity(node *sourceNode) any { + if node == nil || node.fields == nil { + return nil + } + identity := map[string]any{} + for _, key := range []string{"id", "name", "role", "type"} { + field, ok := node.fields[key] + if !ok || !scalarValue(field.value) { + continue + } + identity[key] = field.value + } + if len(identity) == 0 { + return nil + } + return identity +} + +func scalarValue(value any) bool { + switch value.(type) { + case string, bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: + return true + default: + return false + } +} + +func stableNodeValue(node *sourceNode, visited map[*sourceNode]bool) any { + if node == nil { + return nil + } + if node.value != nil { + return node.value + } + if visited[node] { + return "" + } + visited[node] = true + defer delete(visited, node) + if node.fields != nil { + value := make(map[string]any, len(node.fields)) + for key, field := range node.fields { + value[key] = stableNodeValue(field, visited) + } + return value + } + if sourceCollection(node) { + value := make([]any, len(node.items)) + for index, item := range node.items { + value[index] = stableNodeValue(item, visited) + } + return value + } + return nil +} + +func selectionFingerprint(value any) (string, error) { + canonical, err := json.Marshal(value) + if err != nil { + return "", fmt.Errorf("encode indexed selection fingerprint: %w", err) + } + sum := sha256.Sum256(canonical) + return fmt.Sprintf("sha256:%x", sum), nil +} + +func formatFilters(filters []pathFilter) string { + parts := make([]string, len(filters)) + for index, filter := range filters { + parts[index] = fmt.Sprintf("%s=%q", filter.key, filter.value) + } + return strings.Join(parts, ",") +} + +func buildProjection( + kind syncdomain.Kind, + mode syncdomain.VariationMode, + selected *sourceNode, + fieldMap map[string]string, +) (syncdomain.Projection, error) { + switch kind { + case syncdomain.KindVariation: + projection := syncdomain.VariationProjection{ + Mode: mode, FieldStates: make(map[syncdomain.Field]syncdomain.FieldState), + } + if !projection.Mode.Valid() { + projection.Mode = syncdomain.VariationModeAgent + } + for name, path := range fieldMap { + node, err := mappedNode(selected, name, path) + if err != nil { + return nil, err + } + field := syncdomain.Field(name) + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.capability, Known: node.value != nil, Reason: node.reason, Code: node.code, File: node.file, + } + parameterKey, parameterField := syncdomain.ModelParameterKey(field) + if parameterField { + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.capability, Known: true, Reason: node.reason, Code: node.code, File: node.file, + } + value, ok := jsonValue(node.value) + if !ok { + return nil, mapTypeError(name, "a JSON-compatible value") + } + if projection.Parameters == nil { + parameters := map[string]any{} + projection.Parameters = ¶meters + } + (*projection.Parameters)[parameterKey] = value + continue + } + if field == syncdomain.FieldModelParameters && node.value == nil { + return nil, mapTypeError(name, "an object") + } + if node.value == nil { + continue + } + switch field { + case syncdomain.FieldPrompt, syncdomain.FieldMessages: + if field == syncdomain.FieldMessages || projection.Mode == syncdomain.VariationModeCompletion { + switch value := node.value.(type) { + case []syncdomain.Message: + for _, message := range value { + if !syncdomain.ValidMessageRole(message.Role) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorMapTypeMismatch, + fmt.Sprintf("message role %q is not supported", message.Role), + "Use system, user, or assistant.", + ) + } + } + projection.Mode = syncdomain.VariationModeCompletion + projection.Messages = &value + case string: + messages := []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: value}} + projection.Messages = &messages + default: + return nil, mapTypeError(name, "messages") + } + } else { + value, ok := node.value.(string) + if !ok { + return nil, mapTypeError(name, "a string") + } + projection.Instructions = &value + } + case syncdomain.FieldName: + value, ok := node.value.(string) + if !ok { + return nil, mapTypeError(name, "a string") + } + projection.Name = &value + case syncdomain.FieldModel: + value, ok := node.value.(string) + if !ok { + return nil, mapTypeError(name, "a string") + } + projection.Model = &value + case syncdomain.FieldModelParameters: + value, ok := objectValue(node.value) + if !ok { + return nil, mapTypeError(name, "an object") + } + value = cloneObject(value) + delete(value, "tools") + projection.Parameters = &value + } + } + _, promptMapped := projection.FieldStates[syncdomain.FieldPrompt] + _, messagesMapped := projection.FieldStates[syncdomain.FieldMessages] + if !promptMapped && !messagesMapped { + return nil, fmt.Errorf("map field %q is required for a variation", syncdomain.FieldPrompt) + } + return projection, nil + case syncdomain.KindTool: + projection := syncdomain.ToolProjection{FieldStates: make(map[syncdomain.Field]syncdomain.FieldState)} + for name, path := range fieldMap { + node, err := mappedNode(selected, name, path) + if err != nil { + return nil, err + } + field := syncdomain.Field(name) + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.capability, Known: node.value != nil, Reason: node.reason, Code: node.code, File: node.file, + } + if node.value == nil { + continue + } + switch field { + case syncdomain.FieldDescription: + value, ok := node.value.(string) + if !ok { + return nil, mapTypeError(name, "a string") + } + projection.Description = &value + case syncdomain.FieldSchema: + value, ok := objectValue(node.value) + if !ok { + return nil, mapTypeError(name, "an object") + } + projection.Schema = &value + case syncdomain.FieldCustomParameters: + value, ok := objectValue(node.value) + if !ok { + return nil, mapTypeError(name, "an object") + } + projection.CustomParameters = &value + case syncdomain.FieldTags: + value, ok := stringList(node.value) + if !ok { + return nil, mapTypeError(name, "a string collection") + } + projection.Tags = &value + } + } + return projection, nil + case syncdomain.KindSkill: + projection := syncdomain.SkillProjection{FieldStates: make(map[syncdomain.Field]syncdomain.FieldState)} + for name, path := range fieldMap { + node, err := mappedNode(selected, name, path) + if err != nil { + return nil, err + } + field := syncdomain.Field(name) + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.capability, Known: node.value != nil, Reason: node.reason, Code: node.code, File: node.file, + } + if node.value == nil { + continue + } + value, ok := node.value.(string) + if !ok { + return nil, mapTypeError(name, "a string") + } + if field == syncdomain.FieldDescription { + projection.Description = &value + } else { + projection.Markdown = &value + } + } + return projection, nil + default: + return nil, fmt.Errorf("unsupported resource kind %q", kind) + } +} + +func mappedNode(root *sourceNode, name, path string) (*sourceNode, error) { + node, err := resolveSourcePath(root, path) + if err != nil { + return nil, fmt.Errorf("map field %q: %w", name, err) + } + return node, nil +} + +func projectionField(projection syncdomain.Projection, field syncdomain.Field) (any, bool, error) { + switch value := projection.(type) { + case syncdomain.VariationProjection: + switch field { + case syncdomain.FieldPrompt, syncdomain.FieldMessages: + if value.Instructions != nil { + return *value.Instructions, true, nil + } + if value.Messages != nil { + return *value.Messages, true, nil + } + case syncdomain.FieldName: + if value.Name != nil { + return *value.Name, true, nil + } + case syncdomain.FieldModel: + if value.Model != nil { + return *value.Model, true, nil + } + case syncdomain.FieldModelParameters: + if value.Parameters != nil { + return cloneObject(*value.Parameters), true, nil + } + case syncdomain.FieldTools: + if value.Tools != nil { + return *value.Tools, true, nil + } + case syncdomain.FieldSkills: + if value.Skills != nil { + return *value.Skills, true, nil + } + default: + if key, parameter := syncdomain.ModelParameterKey(field); parameter && value.Parameters != nil { + item, present := (*value.Parameters)[key] + return item, present, nil + } + } + case syncdomain.ToolProjection: + switch field { + case syncdomain.FieldDescription: + if value.Description != nil { + return *value.Description, true, nil + } + case syncdomain.FieldSchema: + if value.Schema != nil { + return *value.Schema, true, nil + } + case syncdomain.FieldCustomParameters: + if value.CustomParameters != nil { + return *value.CustomParameters, true, nil + } + case syncdomain.FieldTags: + if value.Tags != nil { + return *value.Tags, true, nil + } + } + case syncdomain.SkillProjection: + if field == syncdomain.FieldDescription && value.Description != nil { + return *value.Description, true, nil + } + if field == syncdomain.FieldMarkdown && value.Markdown != nil { + return *value.Markdown, true, nil + } + default: + return nil, false, fmt.Errorf("unsupported projection %T", projection) + } + return nil, false, nil +} + +func parseSourceBundle(bundle SourceBundle, format string) (*sourceNode, error) { + adapter, err := registeredAdapter(format) + if err != nil { + return nil, err + } + inspection, err := adapter.Inspect(bundle) + if err != nil { + if _, ok := syncdomain.ProblemFrom(err); !ok { + err = syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + err.Error(), + "Fix the source syntax or use a supported static form.", + ).WithFile(bundle.Primary.Path, 0, 0), err) + } + } + markSourceFile(inspection.root, bundle.Primary.Path) + return inspection.root, err +} + +func markSourceFile(node *sourceNode, file string) { + markSourceFileOnce(node, file, map[*sourceNode]struct{}{}) +} + +func markSourceFileOnce(node *sourceNode, file string, visited map[*sourceNode]struct{}) { + if node == nil { + return + } + if _, seen := visited[node]; seen { + return + } + visited[node] = struct{}{} + if node.file == "" { + node.file = file + } + for _, child := range node.fields { + markSourceFileOnce(child, file, visited) + } + for _, child := range node.items { + markSourceFileOnce(child, file, visited) + } +} + +func fieldAllowed(kind syncdomain.Kind, field syncdomain.Field) bool { + switch kind { + case syncdomain.KindVariation: + if _, parameter := syncdomain.ModelParameterKey(field); parameter { + return true + } + return slices.Contains([]syncdomain.Field{ + syncdomain.FieldPrompt, syncdomain.FieldMessages, syncdomain.FieldName, syncdomain.FieldModel, + syncdomain.FieldModelParameters, + }, field) + case syncdomain.KindTool: + return slices.Contains([]syncdomain.Field{ + syncdomain.FieldDescription, syncdomain.FieldSchema, + syncdomain.FieldCustomParameters, syncdomain.FieldTags, + }, field) + case syncdomain.KindSkill: + return field == syncdomain.FieldDescription || field == syncdomain.FieldMarkdown + default: + return false + } +} + +func semanticEqual(left, right any) bool { + leftJSON, leftErr := json.Marshal(left) + rightJSON, rightErr := json.Marshal(right) + return leftErr == nil && rightErr == nil && bytes.Equal(leftJSON, rightJSON) +} + +func mapTypeError(field, expected string) error { + return syncdomain.NewProblem( + syncdomain.ErrorMapTypeMismatch, + fmt.Sprintf("mapped field %q needs %s", field, expected), + "Map the field to a compatible static value.", + ) +} + +func stringList(value any) ([]string, bool) { + switch values := value.(type) { + case []string: + return slices.Clone(values), true + case []any: + result := make([]string, len(values)) + for index, value := range values { + text, ok := value.(string) + if !ok { + return nil, false + } + result[index] = text + } + return result, true + default: + return nil, false + } +} + +func attachmentKeys(refs []syncdomain.AttachmentRef) []string { + keys := make([]string, len(refs)) + for index := range refs { + keys[index] = refs[index].Key + } + slices.Sort(keys) + return keys +} + +func objectValue(value any) (map[string]any, bool) { + object, ok := value.(map[string]any) + if !ok { + return nil, false + } + return object, true +} + +func cloneObject(value map[string]any) map[string]any { + cloned, ok := jsonValue(value) + if !ok { + return map[string]any{} + } + object, ok := cloned.(map[string]any) + if !ok { + return map[string]any{} + } + return object +} + +func jsonValue(value any) (any, bool) { + encoded, err := json.Marshal(value) + if err != nil { + return nil, false + } + var decoded any + if err := json.Unmarshal(encoded, &decoded); err != nil { + return nil, false + } + return decoded, true +} + +func stringNode(value string, start, end int, encode func(string) ([]byte, error)) *sourceNode { + return &sourceNode{ + value: value, start: start, end: end, capability: CapabilityDirect, + encode: func(value any) ([]byte, error) { + text, ok := value.(string) + if !ok { + return nil, mapTypeError("source", "a string") + } + return encode(text) + }, + } +} + +func normalizeMarkdown(text string) string { + return strings.TrimSpace(strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n")) +} diff --git a/internal/sync/reference/source_test.go b/internal/sync/reference/source_test.go new file mode 100644 index 000000000..73dbc2adc --- /dev/null +++ b/internal/sync/reference/source_test.go @@ -0,0 +1,120 @@ +package reference + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestPathParserReadsTwoFieldFilter(t *testing.T) { + path, err := parsePath(`items[id="one",role="system"]`) + require.NoError(t, err) + require.Len(t, path.steps, 1) + assert.Equal(t, "items", path.steps[0].key) + assert.Equal(t, []pathFilter{{key: "id", value: "one"}, {key: "role", value: "system"}}, path.steps[0].filters) +} + +func TestResolvePathSupportsChainedAndMixedIndexes(t *testing.T) { + root := &sourceNode{fields: map[string]*sourceNode{ + "agents": {items: []*sourceNode{ + {fields: map[string]*sourceNode{ + "id": {value: "support"}, + "messages": {items: []*sourceNode{ + {fields: map[string]*sourceNode{"content": {value: "First"}}}, + {fields: map[string]*sourceNode{"content": {value: "Second"}}}, + }, collection: true}, + }}, + }, collection: true}, + }} + + value, err := resolveSourcePath(root, `agents[id="support"].messages[1].content`) + + require.NoError(t, err) + assert.Equal(t, "Second", value.value) +} + +func TestResolvePathIndexErrorsUseStableCodes(t *testing.T) { + root := &sourceNode{fields: map[string]*sourceNode{ + "items": {items: []*sourceNode{{value: "only"}}, collection: true}, + "value": {value: "text"}, + }} + tests := []struct { + path string + code syncdomain.ErrorCode + }{ + {path: "items[1]", code: syncdomain.ErrorIndexOutOfRange}, + {path: "value[0]", code: syncdomain.ErrorPathTypeMismatch}, + } + for _, test := range tests { + _, err := resolveSourcePath(root, test.path) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, test.code, problem.Code) + } +} + +func TestIndexedSelectionIdentityUsesStableFieldsAndFallback(t *testing.T) { + identified := &sourceNode{fields: map[string]*sourceNode{ + "id": {value: "support"}, + "n": {value: 1}, + }} + assert.Equal(t, map[string]any{"id": "support"}, directSelectionIdentity(identified)) + assert.Nil(t, directSelectionIdentity(nil)) + assert.True(t, scalarValue(uint64(2))) + assert.False(t, scalarValue(nil)) + assert.False(t, sourceCollection(nil)) + assert.True(t, sourceCollection(&sourceNode{value: []string{"one"}})) + + cycle := &sourceNode{fields: map[string]*sourceNode{}} + cycle.fields["self"] = cycle + fallback := &sourceNode{fields: map[string]*sourceNode{"prompt": {value: "Help"}}} + selections, err := indexedSelections( + []indexedResolution{ + {path: "items[0]", collectionLength: 1, item: identified}, + {path: "items[0].children[0]", collectionLength: 1, item: fallback}, + {path: "items[0].cycle[0]", collectionLength: 1, item: cycle}, + }, + fallback, + ) + require.NoError(t, err) + require.Len(t, selections, 3) + for _, selection := range selections { + assert.NotEmpty(t, selection.Fingerprint) + } +} + +func TestMarkdownSectionUpdatePreservesOtherSections(t *testing.T) { + source := []byte("# Support\nOld prompt\n\n# Other\nKeep this.\n") + id := syncdomain.VariationID("project", "config", "default") + reference := syncdomain.Reference{ + File: "AGENTS.md", Format: "markdown", + Select: `sections[heading="Support"]`, Map: map[string]string{"prompt": "body"}, + } + prompt := "New prompt" + result, err := UpdateSource( + sourceBundle("AGENTS.md", source), + id, + syncdomain.VariationModeAgent, + reference, + syncdomain.VariationProjection{Instructions: &prompt}, + ) + require.NoError(t, err) + updated := applyConformanceEdits(t, source, directEdits(t, result)) + assert.Contains(t, string(updated), "# Other\nKeep this.") +} + +func FuzzPathParserNeverPanics(f *testing.F) { + for _, seed := range []string{ + "$", "field.child", `items[id="one"]`, `object["literal.key"]`, + "items[0]", "items[-1]", "items[1.5]", "items[]", "items[2147483648]", + "items[0][1]", `items[id="support"].messages[0]`, + } { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, path string) { + _ = ParsePath(path) + }) +} diff --git a/internal/sync/reference/testdata/agent-skill/refunds/references/policy.md b/internal/sync/reference/testdata/agent-skill/refunds/references/policy.md new file mode 100644 index 000000000..52b962517 --- /dev/null +++ b/internal/sync/reference/testdata/agent-skill/refunds/references/policy.md @@ -0,0 +1,3 @@ +# Refund policy + +Escalate refunds above $500. diff --git a/internal/sync/reference/testdata/agent-skill/refunds/scripts/check_refund.py b/internal/sync/reference/testdata/agent-skill/refunds/scripts/check_refund.py new file mode 100644 index 000000000..e28fe6cd2 --- /dev/null +++ b/internal/sync/reference/testdata/agent-skill/refunds/scripts/check_refund.py @@ -0,0 +1,2 @@ +def eligible(total: float) -> bool: + return total <= 500 diff --git a/internal/sync/reference/testdata/agent-skill/refunds/skill-instructions.md.golden b/internal/sync/reference/testdata/agent-skill/refunds/skill-instructions.md.golden new file mode 100644 index 000000000..95237b040 --- /dev/null +++ b/internal/sync/reference/testdata/agent-skill/refunds/skill-instructions.md.golden @@ -0,0 +1,12 @@ +--- +name: refunds +description: Explain the refund policy and escalation rules. +license: Apache-2.0 +metadata: + owner: support-platform +--- + +# Refund handling + +Confirm the order ID first. +Escalate refunds above $500. diff --git a/internal/sync/reference/testdata/markdown/agents-instructions.md.golden b/internal/sync/reference/testdata/markdown/agents-instructions.md.golden new file mode 100644 index 000000000..16f2d0c47 --- /dev/null +++ b/internal/sync/reference/testdata/markdown/agents-instructions.md.golden @@ -0,0 +1,9 @@ +# Repository guide + +## Support agent + +Ask for an order ID before discussing an order. + +## Build + +Run `make test`. From 40c351dfc6b20f779b897b953415a35086e33b6d Mon Sep 17 00:00:00 2001 From: Cliff Tawiah Date: Sun, 1 Nov 2026 20:42:58 -0500 Subject: [PATCH 4/5] feat(sync): add structured document adapters --- .../reference/adapter_conformance_test.go | 18 + internal/sync/reference/defensive_test.go | 35 ++ internal/sync/reference/document.go | 8 +- internal/sync/reference/json.go | 238 ++++++++++ internal/sync/reference/parameters_test.go | 203 ++++++++ internal/sync/reference/source.go | 34 ++ internal/sync/reference/structured_test.go | 440 ++++++++++++++++++ .../sync/reference/testdata/json/mcp.json | 32 ++ .../reference/testdata/toml/pyproject.toml | 20 + .../sync/reference/testdata/yaml/agents.yaml | 16 + internal/sync/reference/toml.go | 307 ++++++++++++ internal/sync/reference/yaml.go | 218 +++++++++ 12 files changed, 1567 insertions(+), 2 deletions(-) create mode 100644 internal/sync/reference/defensive_test.go create mode 100644 internal/sync/reference/json.go create mode 100644 internal/sync/reference/parameters_test.go create mode 100644 internal/sync/reference/structured_test.go create mode 100644 internal/sync/reference/testdata/json/mcp.json create mode 100644 internal/sync/reference/testdata/toml/pyproject.toml create mode 100644 internal/sync/reference/testdata/yaml/agents.yaml create mode 100644 internal/sync/reference/toml.go create mode 100644 internal/sync/reference/yaml.go diff --git a/internal/sync/reference/adapter_conformance_test.go b/internal/sync/reference/adapter_conformance_test.go index 22246424c..0eee2569a 100644 --- a/internal/sync/reference/adapter_conformance_test.go +++ b/internal/sync/reference/adapter_conformance_test.go @@ -35,6 +35,24 @@ func TestRegisteredFormatsReadAndWriteRealisticSources(t *testing.T) { id: syncdomain.ResourceID{Kind: syncdomain.KindSkill, ProjectKey: "project", LookupKey: "refunds"}, want: "Explain the refund policy and escalation rules.", update: "Explain refunds and escalation rules.", }, + { + name: "JSON tool", fixture: "json/mcp.json", format: "json", + id: syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "lookup_order"}, + selectPath: `tools[name="lookup_order"]`, fieldMap: map[string]string{"description": "description"}, + want: "Look up one order.", update: "Look up one order and status.", + }, + { + name: "YAML agent", fixture: "yaml/agents.yaml", format: "yaml", + id: syncdomain.VariationID("project", "support", "default"), + selectPath: `agents[name="support"]`, fieldMap: map[string]string{"prompt": "prompt"}, + want: "Ask for an order ID.", update: "Ask for an order ID and region.", + }, + { + name: "TOML agent", fixture: "toml/pyproject.toml", format: "toml", + id: syncdomain.VariationID("project", "support", "default"), + selectPath: "tool.agent.support", fieldMap: map[string]string{"prompt": "prompt"}, + want: "Ask for an order ID.\nKeep the answer short.", update: "Ask for an order ID and region.", + }, } for _, test := range tests { diff --git a/internal/sync/reference/defensive_test.go b/internal/sync/reference/defensive_test.go new file mode 100644 index 000000000..064511af6 --- /dev/null +++ b/internal/sync/reference/defensive_test.go @@ -0,0 +1,35 @@ +package reference + +import ( + "testing" + + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestStructuredAdaptersRejectMalformedSources(t *testing.T) { + tests := []struct { + name, format, source string + }{ + {name: "JSON duplicate", format: "json", source: `{"key":1,"key":2}`}, + {name: "JSON trailing", format: "json", source: `{"key":1} trailing`}, + {name: "YAML duplicate", format: "yaml", source: "key: one\nkey: two\n"}, + {name: "TOML duplicate", format: "toml", source: "key = 1\nkey = 2\n"}, + } + id := syncdomain.VariationID("project", "config", "default") + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := ParseSource( + sourceBundle("source", []byte(test.source)), + id, + syncdomain.VariationModeAgent, + syncdomain.Reference{ + File: "source", Format: test.format, Select: "$", + Map: map[string]string{"prompt": "key"}, + }, + ) + require.Error(t, err) + }) + } +} diff --git a/internal/sync/reference/document.go b/internal/sync/reference/document.go index 3e888d854..fc3d55b28 100644 --- a/internal/sync/reference/document.go +++ b/internal/sync/reference/document.go @@ -370,8 +370,12 @@ func quoteYAML(value, previous string) string { func encodeYAMLString(value, previous string) ([]byte, error) { for _, character := range value { - if unicode.IsControl(character) && character != '\n' && character != '\r' && character != '\t' { - return nil, fmt.Errorf("YAML string contains unsupported control character U+%04X", character) + valid := character == '\n' || character == '\r' || character == '\t' || + character >= 0x20 && character <= 0xd7ff || + character >= 0xe000 && character <= 0xfffd || + character >= 0x10000 && character <= utf8.MaxRune + if !valid || unicode.IsControl(character) && character != '\n' && character != '\r' && character != '\t' { + return nil, fmt.Errorf("YAML string contains unsupported character U+%04X", character) } } return []byte(quoteYAML(value, previous)), nil diff --git a/internal/sync/reference/json.go b/internal/sync/reference/json.go new file mode 100644 index 000000000..98a1b1b30 --- /dev/null +++ b/internal/sync/reference/json.go @@ -0,0 +1,238 @@ +package reference + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "unicode/utf8" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type jsonAdapter struct{} + +func (jsonAdapter) Format() string { return "json" } +func (jsonAdapter) Inspect(bundle SourceBundle) (Inspection, error) { + root, err := parseJSONDocument(bundle.Primary) + return Inspection{root: root}, err +} +func (jsonAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "$", nil } + +func init() { mustRegister(jsonAdapter{}) } + +type jsonParser struct { + data []byte + offset int +} + +func parseJSONDocument(document Document) (*sourceNode, error) { + if !utf8.Valid(document.Data) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "JSON source is not valid UTF-8", + "Save the file as UTF-8.", + ).WithFile(document.Path, 0, 0) + } + parser := jsonParser{data: document.Data} + if bytes.HasPrefix(parser.data, []byte("\ufeff")) { + parser.offset = len([]byte("\ufeff")) + } + parser.skipSpace() + root, err := parser.value() + if err != nil { + return nil, err + } + parser.skipSpace() + if parser.offset != len(parser.data) { + return nil, parser.problem("JSON source has trailing input") + } + return root, nil +} + +func (parser *jsonParser) value() (*sourceNode, error) { + parser.skipSpace() + if parser.offset >= len(parser.data) { + return nil, parser.problem("JSON value is missing") + } + switch parser.data[parser.offset] { + case '{': + return parser.object() + case '[': + return parser.array() + case '"': + return parser.string() + case 't': + return parser.keyword("true", true) + case 'f': + return parser.keyword("false", false) + case 'n': + return parser.keyword("null", nil) + default: + return parser.number() + } +} + +func (parser *jsonParser) object() (*sourceNode, error) { + start := parser.offset + parser.offset++ + parser.skipSpace() + fields := map[string]*sourceNode{} + value := map[string]any{} + if parser.take('}') { + return jsonObjectNode(value, fields, start, parser.offset), nil + } + for { + keyNode, err := parser.string() + if err != nil { + return nil, err + } + key := keyNode.value.(string) + if _, duplicate := fields[key]; duplicate { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("JSON object contains duplicate key %q", key), + "Remove the duplicate key.", + ) + } + parser.skipSpace() + if !parser.take(':') { + return nil, parser.problem("JSON object key needs a colon") + } + item, err := parser.value() + if err != nil { + return nil, err + } + fields[key], value[key] = item, item.value + parser.skipSpace() + if parser.take('}') { + return jsonObjectNode(value, fields, start, parser.offset), nil + } + if !parser.take(',') { + return nil, parser.problem("JSON object needs a comma or closing brace") + } + parser.skipSpace() + } +} + +func (parser *jsonParser) array() (*sourceNode, error) { + start := parser.offset + parser.offset++ + parser.skipSpace() + var items []*sourceNode + var values []any + if parser.take(']') { + return jsonArrayNode(values, items, start, parser.offset), nil + } + for { + item, err := parser.value() + if err != nil { + return nil, err + } + items, values = append(items, item), append(values, item.value) + parser.skipSpace() + if parser.take(']') { + return jsonArrayNode(values, items, start, parser.offset), nil + } + if !parser.take(',') { + return nil, parser.problem("JSON array needs a comma or closing bracket") + } + parser.skipSpace() + } +} + +func (parser *jsonParser) string() (*sourceNode, error) { + start := parser.offset + if !parser.take('"') { + return nil, parser.problem("JSON string must start with a quote") + } + escaped := false + for parser.offset < len(parser.data) { + next := parser.data[parser.offset] + parser.offset++ + switch { + case escaped: + escaped = false + case next == '\\': + escaped = true + case next == '"': + raw := parser.data[start:parser.offset] + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return nil, parser.problem("JSON string is invalid") + } + return stringNode(value, start, parser.offset, func(text string) ([]byte, error) { + return json.Marshal(text) + }), nil + case next < 0x20: + return nil, parser.problem("JSON string contains a control character") + } + } + return nil, parser.problem("JSON string is not closed") +} + +func (parser *jsonParser) number() (*sourceNode, error) { + start := parser.offset + for parser.offset < len(parser.data) && bytes.ContainsRune([]byte("-+0123456789.eE"), rune(parser.data[parser.offset])) { + parser.offset++ + } + if start == parser.offset { + return nil, parser.problem("JSON value is invalid") + } + raw := parser.data[start:parser.offset] + var value float64 + if err := json.Unmarshal(raw, &value); err != nil { + return nil, parser.problem("JSON number is invalid") + } + return numberNode(value, start, parser.offset, func(number float64) ([]byte, error) { + return []byte(strconv.FormatFloat(number, 'g', -1, 64)), nil + }), nil +} + +func (parser *jsonParser) keyword(keyword string, value any) (*sourceNode, error) { + start := parser.offset + if !bytes.HasPrefix(parser.data[start:], []byte(keyword)) { + return nil, parser.problem("JSON value is invalid") + } + parser.offset += len(keyword) + return &sourceNode{ + value: value, start: start, end: parser.offset, capability: CapabilityDirect, + encode: func(next any) ([]byte, error) { return json.Marshal(next) }, + }, nil +} + +func (parser *jsonParser) skipSpace() { + for parser.offset < len(parser.data) && bytes.ContainsRune([]byte(" \t\r\n"), rune(parser.data[parser.offset])) { + parser.offset++ + } +} + +func (parser *jsonParser) take(expected byte) bool { + if parser.offset >= len(parser.data) || parser.data[parser.offset] != expected { + return false + } + parser.offset++ + return true +} + +func (parser *jsonParser) problem(message string) error { + return syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + fmt.Sprintf("%s at byte %d", message, parser.offset), + "Fix the JSON syntax.", + ) +} + +func jsonObjectNode(value map[string]any, fields map[string]*sourceNode, start, end int) *sourceNode { + return &sourceNode{ + value: value, fields: fields, start: start, end: end, capability: CapabilityDirect, + encode: func(next any) ([]byte, error) { return json.Marshal(next) }, + } +} + +func jsonArrayNode(value []any, items []*sourceNode, start, end int) *sourceNode { + return &sourceNode{ + value: value, items: items, collection: true, start: start, end: end, capability: CapabilityDirect, + encode: func(next any) ([]byte, error) { return json.Marshal(next) }, + } +} diff --git a/internal/sync/reference/parameters_test.go b/internal/sync/reference/parameters_test.go new file mode 100644 index 000000000..83ab0b842 --- /dev/null +++ b/internal/sync/reference/parameters_test.go @@ -0,0 +1,203 @@ +package reference + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestJSONIndividualModelParameterMappingsRoundTrip(t *testing.T) { + tests := []struct { + name string + key string + source any + replacement any + }{ + {name: "temperature", key: "temperature", source: 0.2, replacement: 0.7}, + {name: "top k", key: "top_k", source: 20, replacement: 40}, + {name: "top p", key: "top_p", source: 0.9, replacement: 0.8}, + {name: "max tokens", key: "max_tokens", source: 256, replacement: 512}, + {name: "future provider key", key: "Provider.Future-Key", source: "v1", replacement: "v2"}, + {name: "string", key: "response_style", source: "brief", replacement: "detailed"}, + {name: "boolean", key: "reasoning", source: true, replacement: false}, + {name: "null", key: "seed", source: nil, replacement: true}, + { + name: "nested object", key: "provider", + source: map[string]any{"mode": "fast"}, + replacement: map[string]any{"mode": "careful", "retry": true}, + }, + { + name: "array", key: "stop", + source: []any{"END"}, + replacement: []any{"DONE", map[string]any{"token": 2}}, + }, + } + id := syncdomain.VariationID("project", "config", "default") + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source, err := json.Marshal(map[string]any{"prompt": "Help", "value": test.source}) + require.NoError(t, err) + target := modelParameterTarget(test.key) + reference := syncdomain.Reference{ + File: "agent.json", Format: "json", + Map: map[string]string{"prompt": "prompt", target: "value"}, + } + + projection, err := ParseSource( + sourceBundle(reference.File, source), id, syncdomain.VariationModeAgent, reference, + ) + require.NoError(t, err) + value, present := syncdomain.ProjectionFieldValue(projection, syncdomain.Field(target)) + assert.True(t, present) + assertJSONValueEqual(t, test.source, value) + + parameters := map[string]any{test.key: test.replacement} + update, err := UpdateSource( + sourceBundle(reference.File, source), id, syncdomain.VariationModeAgent, reference, + syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Parameters: ¶meters, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.Field(target): {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + ) + require.NoError(t, err) + require.Len(t, update.Fields[syncdomain.Field(target)].Edits, 1) + updated := applyParameterEdits(source, update.Fields[syncdomain.Field(target)].Edits...) + reparsed, err := ParseSource( + sourceBundle(reference.File, updated), id, syncdomain.VariationModeAgent, reference, + ) + require.NoError(t, err) + value, present = syncdomain.ProjectionFieldValue(reparsed, syncdomain.Field(target)) + assert.True(t, present) + assertJSONValueEqual(t, test.replacement, value) + }) + } +} + +func TestJSONWholeModelParametersOwnsCanonicalMap(t *testing.T) { + source := []byte(`{"prompt":"Help","parameters":{"temperature":0.2,"tools":[{"name":"search"}]}}`) + id := syncdomain.VariationID("project", "config", "default") + reference := syncdomain.Reference{ + File: "agent.json", Format: "json", + Map: map[string]string{"prompt": "prompt", "model.parameters": "parameters"}, + } + projection, err := ParseSource( + sourceBundle(reference.File, source), id, syncdomain.VariationModeAgent, reference, + ) + require.NoError(t, err) + parameters := projection.(syncdomain.VariationProjection).Parameters + require.NotNil(t, parameters) + assert.Equal(t, map[string]any{"temperature": 0.2}, *parameters) + + replacement := map[string]any{ + "top_k": 30, + "nested": map[string]any{"enabled": true}, + "stop": []any{"END"}, + "seed": nil, + } + update, err := UpdateSource( + sourceBundle(reference.File, source), id, syncdomain.VariationModeAgent, reference, + syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Parameters: &replacement, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldModelParameters: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + ) + require.NoError(t, err) + require.Len(t, update.Fields[syncdomain.FieldModelParameters].Edits, 1) + updated := applyParameterEdits(source, update.Fields[syncdomain.FieldModelParameters].Edits...) + assert.JSONEq(t, `{"prompt":"Help","parameters":{"top_k":30,"nested":{"enabled":true},"stop":["END"],"seed":null}}`, string(updated)) +} + +func TestNormalizeReferenceRejectsOverlappingOrInvalidParameterTargets(t *testing.T) { + id := syncdomain.VariationID("project", "config", "default") + tests := []struct { + name string + targets map[string]string + code syncdomain.ErrorCode + message string + }{ + { + name: "whole and child overlap", + targets: map[string]string{ + "prompt": "prompt", "model.parameters": "parameters", + `model.parameters["temperature"]`: "temperature", + }, + code: syncdomain.ErrorInvalidReference, message: "overlaps", + }, + { + name: "same exact key through JSON escape", + targets: map[string]string{ + "prompt": "prompt", `model.parameters["top_k"]`: "top_k", + `model.parameters["top\u005fk"]`: "other", + }, + code: syncdomain.ErrorInvalidReference, message: "same model parameter", + }, + { + name: "expanded tools child", + targets: map[string]string{ + "prompt": "prompt", `model.parameters["tools"]`: "tools", + }, + code: syncdomain.ErrorInvalidReference, message: "duplicates the tools field", + }, + { + name: "child needs exact bracket key", + targets: map[string]string{ + "prompt": "prompt", "model.parameters.temperature": "temperature", + }, + code: syncdomain.ErrorInvalidReference, message: "unsupported mapped field", + }, + { + name: "target uses shared path syntax", + targets: map[string]string{ + "prompt": "prompt", `model.parameters["temperature"`: "temperature", + }, + code: syncdomain.ErrorInvalidPathSyntax, message: "invalid path", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := NormalizeReference(id, syncdomain.Reference{ + File: "agent.json", Format: "json", Map: test.targets, + }) + require.Error(t, err) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, test.code, problem.Code) + assert.ErrorContains(t, err, test.message) + }) + } +} + +func modelParameterTarget(key string) string { + encoded, _ := json.Marshal(key) + return fmt.Sprintf("model.parameters[%s]", encoded) +} + +func assertJSONValueEqual(t *testing.T, expected, actual any) { + t.Helper() + expectedJSON, err := json.Marshal(expected) + require.NoError(t, err) + actualJSON, err := json.Marshal(actual) + require.NoError(t, err) + assert.JSONEq(t, string(expectedJSON), string(actualJSON)) +} + +func applyParameterEdits(source []byte, edits ...TextEdit) []byte { + updated := append([]byte(nil), source...) + for _, edit := range edits { + next := make([]byte, 0, len(updated)-edit.End+edit.Start+len(edit.Replacement)) + next = append(next, updated[:edit.Start]...) + next = append(next, edit.Replacement...) + next = append(next, updated[edit.End:]...) + updated = next + } + return updated +} diff --git a/internal/sync/reference/source.go b/internal/sync/reference/source.go index 57faa027c..3071acc51 100644 --- a/internal/sync/reference/source.go +++ b/internal/sync/reference/source.go @@ -871,6 +871,10 @@ func projectionField(projection syncdomain.Projection, field syncdomain.Field) ( return nil, false, nil } +func parseSourceDocument(document Document, format string) (*sourceNode, error) { + return parseSourceBundle(SourceBundle{Primary: document}, format) +} + func parseSourceBundle(bundle SourceBundle, format string) (*sourceNode, error) { adapter, err := registeredAdapter(format) if err != nil { @@ -949,6 +953,19 @@ func mapTypeError(field, expected string) error { ) } +func sourceNumber(value any) (float64, bool) { + switch value := value.(type) { + case float64: + return value, true + case int: + return float64(value), true + case int64: + return float64(value), true + default: + return 0, false + } +} + func stringList(value any) ([]string, bool) { switch values := value.(type) { case []string: @@ -1009,6 +1026,10 @@ func jsonValue(value any) (any, bool) { return decoded, true } +func readOnlyNode(start, end int, reason string) *sourceNode { + return &sourceNode{start: start, end: end, capability: CapabilityReadOnly, reason: reason} +} + func stringNode(value string, start, end int, encode func(string) ([]byte, error)) *sourceNode { return &sourceNode{ value: value, start: start, end: end, capability: CapabilityDirect, @@ -1022,6 +1043,19 @@ func stringNode(value string, start, end int, encode func(string) ([]byte, error } } +func numberNode(value float64, start, end int, encode func(float64) ([]byte, error)) *sourceNode { + return &sourceNode{ + value: value, start: start, end: end, capability: CapabilityDirect, + encode: func(value any) ([]byte, error) { + number, ok := sourceNumber(value) + if !ok { + return nil, mapTypeError("source", "a number") + } + return encode(number) + }, + } +} + func normalizeMarkdown(text string) string { return strings.TrimSpace(strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n")) } diff --git a/internal/sync/reference/structured_test.go b/internal/sync/reference/structured_test.go new file mode 100644 index 000000000..6a3e3e737 --- /dev/null +++ b/internal/sync/reference/structured_test.go @@ -0,0 +1,440 @@ +package reference + +import ( + "encoding/json" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestStructuredPathsReadNestedListsAndLiteralKeys(t *testing.T) { + tests := []struct { + name string + format string + source string + path string + want any + }{ + { + name: "JSON keyed list", format: "json", + source: `{"items":[{"name":"one","enabled":false},{"name":"two","enabled":true}]}`, + path: `items[name="two"].enabled`, want: true, + }, + { + name: "YAML nested field", format: "yaml", + source: "agent:\n model:\n temperature: 0.3\n", + path: "agent.model.temperature", want: 0.3, + }, + { + name: "TOML literal key", format: "toml", + source: "[metadata]\n\"display.name\" = \"Support\"\n", + path: `metadata["display.name"]`, want: "Support", + }, + { + name: "TOML inline table list", format: "toml", + source: "items = [{ name = \"one\", prompt = \"First\" }, { name = \"two\", prompt = \"Second\" }]\n", + path: `items[name="two"].prompt`, want: "Second", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root, err := parseSourceDocument(Document{Path: "source", Data: []byte(test.source)}, test.format) + require.NoError(t, err) + value, err := resolveSourcePath(root, test.path) + require.NoError(t, err) + assert.Equal(t, test.want, value.value) + }) + } +} + +func FuzzJSONStringCodec(f *testing.F) { + for _, value := range []string{"plain", "quotes \" and slash \\", "line one\nline two", "Unicode 🌍"} { + f.Add(value) + } + f.Fuzz(func(t *testing.T, value string) { + if !utf8.ValidString(value) { + t.Skip() + } + quoted, err := json.Marshal(value) + require.NoError(t, err) + source := append([]byte(`{"value":`), quoted...) + source = append(source, '}') + root, err := parseSourceDocument(Document{Path: "value.json", Data: source}, "json") + require.NoError(t, err) + node, err := resolveSourcePath(root, "value") + require.NoError(t, err) + encoded, err := node.encode(node.value) + require.NoError(t, err) + var decoded string + require.NoError(t, json.Unmarshal(encoded, &decoded)) + require.Equal(t, value, decoded) + }) +} + +func FuzzYAMLStringCodec(f *testing.F) { + for _, value := range []string{"plain", "quote ' and slash \\", "line one\nline two", "Unicode 🌍"} { + f.Add(value) + } + f.Fuzz(func(t *testing.T, value string) { + if !utf8.ValidString(value) { + t.Skip() + } + for _, previous := range []string{"plain", "'plain'", `"plain"`} { + encoded, err := encodeYAMLString(value, previous) + if err != nil { + t.Skip() + } + source := append([]byte("value: "), encoded...) + source = append(source, '\n') + root, err := parseSourceDocument(Document{Path: "value.yaml", Data: source}, "yaml") + require.NoError(t, err) + node, err := resolveSourcePath(root, "value") + require.NoError(t, err) + require.Equal(t, value, node.value) + } + }) +} + +func TestYAMLStringCodecRejectsUnsupportedCodePoint(t *testing.T) { + _, err := encodeYAMLString("\uffff", "plain") + require.ErrorContains(t, err, "unsupported character U+FFFF") +} + +func FuzzTOMLStringCodec(f *testing.F) { + for _, value := range []string{"plain", "quotes \" and slash \\", "line one\nline two", "Unicode 🌍"} { + f.Add(value) + } + f.Fuzz(func(t *testing.T, value string) { + if !utf8.ValidString(value) { + t.Skip() + } + encoded, err := encodeTOMLString(value, `"plain"`) + require.NoError(t, err) + source := append([]byte("value = "), encoded...) + source = append(source, '\n') + root, err := parseSourceDocument(Document{Path: "value.toml", Data: source}, "toml") + require.NoError(t, err) + node, err := resolveSourcePath(root, "value") + require.NoError(t, err) + require.Equal(t, value, node.value) + }) +} + +func TestStructuredAdaptersRejectDuplicateKeys(t *testing.T) { + tests := []struct { + name string + format string + source string + }{ + {name: "JSON", format: "json", source: `{"name":"first","name":"second"}`}, + {name: "YAML", format: "yaml", source: "name: first\nname: second\n"}, + {name: "TOML", format: "toml", source: "name = \"first\"\nname = \"second\"\n"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := parseSourceDocument(Document{Path: "source", Data: []byte(test.source)}, test.format) + require.Error(t, err) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorDuplicateKey, problem.Code) + }) + } +} + +func TestStructuredAdaptersRejectInvalidSyntax(t *testing.T) { + tests := []struct { + name, format, source string + }{ + {name: "empty JSON", format: "json", source: ""}, + {name: "JSON trailing input", format: "json", source: `{"a":1} true`}, + {name: "JSON missing colon", format: "json", source: `{"a" 1}`}, + {name: "JSON missing comma", format: "json", source: `[1 2]`}, + {name: "JSON unclosed string", format: "json", source: `{"a":"value}`}, + {name: "invalid YAML", format: "yaml", source: "value: [\n"}, + {name: "empty YAML", format: "yaml", source: ""}, + {name: "invalid TOML", format: "toml", source: "value = [\n"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := parseSourceDocument( + Document{Path: "source", Data: []byte(test.source)}, test.format, + ) + require.Error(t, err) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorInvalidSource, problem.Code) + }) + } +} + +func TestYAMLUnsafeFormsAreReadOnly(t *testing.T) { + tests := []struct { + name string + source string + path string + }{ + {name: "block scalar", source: "prompt: |\n Help the user.\n", path: "prompt"}, + {name: "anchor", source: "prompt: &shared \"Help\"\n", path: "prompt"}, + {name: "alias", source: "base: &shared \"Help\"\nprompt: *shared\n", path: "prompt"}, + {name: "merge", source: "base: &base\n prompt: Help\nagent:\n <<: *base\n", path: "agent.prompt"}, + {name: "tag", source: "prompt: !custom Help\n", path: "prompt"}, + {name: "multiple documents", source: "prompt: First\n---\nprompt: Second\n", path: "prompt"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root, err := parseSourceDocument(Document{Path: "agents.yaml", Data: []byte(test.source)}, "yaml") + require.NoError(t, err) + value, err := resolveSourcePath(root, test.path) + require.NoError(t, err) + assert.Equal(t, CapabilityReadOnly, value.capability) + assert.NotEmpty(t, value.reason) + }) + } +} + +func TestTOMLArrayTableFieldsAreReadOnly(t *testing.T) { + source := `[[tool.plugins]] +name = "audit" +enabled = true +` + root, err := parseSourceDocument(Document{Path: "pyproject.toml", Data: []byte(source)}, "toml") + require.NoError(t, err) + + value, err := resolveSourcePath(root, `tool.plugins[name="audit"].enabled`) + + require.NoError(t, err) + assert.Equal(t, CapabilityReadOnly, value.capability) +} + +func TestStructuredEditsPreserveBOMAndCRLFOutsideSpan(t *testing.T) { + source := []byte("\ufeff{\r\n \"description\": \"Old\",\r\n \"enabled\": true\r\n}\r\n") + id := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "tool"} + reference := syncdomain.Reference{ + File: "tool.json", Format: "json", Select: "$", + Map: map[string]string{"description": "description"}, + } + description := "New 🌍" + + result, err := UpdateSource( + sourceBundle(reference.File, source), id, "", reference, + syncdomain.ToolProjection{Description: &description}, + ) + + require.NoError(t, err) + updated := applyConformanceEdits(t, source, directEdits(t, result)) + assert.Contains(t, string(updated), "\ufeff{\r\n") + assert.Contains(t, string(updated), "\"enabled\": true\r\n") + assert.Contains(t, string(updated), `"description": "New 🌍"`) +} + +func TestStructuredIndexWriteRoundTripsPreserveOtherItems(t *testing.T) { + tests := []struct { + name, format, source, unchanged string + }{ + { + name: "JSON", format: "json", + source: "{\n \"tools\": [{\"name\":\"first\",\"description\":\"Keep\"}, {\"name\":\"second\",\"description\":\"Old\"}]\n}\n", + unchanged: `{"name":"first","description":"Keep"}`, + }, + { + name: "YAML", format: "yaml", + source: "tools:\n - {name: first, description: Keep}\n - {name: second, description: Old}\n", + unchanged: "{name: first, description: Keep}", + }, + { + name: "TOML", format: "toml", + source: "tools = [{ name = \"first\", description = \"Keep\" }, { name = \"second\", description = \"Old\" }]\n", + unchanged: `{ name = "first", description = "Keep" }`, + }, + } + id := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "second"} + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source := []byte(test.source) + reference := syncdomain.Reference{ + File: "tools." + test.format, Format: test.format, Select: "tools[1]", + Map: map[string]string{"description": "description"}, + } + projection, selections, err := ParseSourceWithSelections(sourceBundle(reference.File, source), id, "", reference) + require.NoError(t, err) + assert.Equal(t, "Old", *projection.(syncdomain.ToolProjection).Description) + require.Len(t, selections, 1) + assert.Equal(t, 2, selections[0].CollectionLength) + + description := "Updated" + update, err := UpdateSource( + sourceBundle(reference.File, source), id, "", reference, + syncdomain.ToolProjection{Description: &description}, + ) + require.NoError(t, err) + updated := applyConformanceEdits(t, source, directEdits(t, update)) + assert.Contains(t, string(updated), test.unchanged) + assert.Contains(t, string(updated), "Updated") + + roundTrip, _, err := ParseSourceWithSelections( + sourceBundle(reference.File, updated), id, "", reference, + ) + require.NoError(t, err) + assert.Equal(t, description, *roundTrip.(syncdomain.ToolProjection).Description) + }) + } +} + +func TestIndexedSelectionsDetectNestedReorders(t *testing.T) { + id := syncdomain.VariationID("project", "completion", "default") + reference := syncdomain.Reference{ + File: "agents.json", Format: "json", Select: "agents[0]", + Map: map[string]string{"prompt": "messages[0].content"}, + } + before := []byte(`{"agents":[ + {"id":"support","messages":[{"role":"system","content":"Help"},{"role":"user","content":"Example"}]}, + {"id":"other","messages":[{"role":"system","content":"Other"}]} + ]}`) + after := []byte(`{"agents":[ + {"id":"other","messages":[{"role":"system","content":"Other"}]}, + {"id":"support","messages":[{"role":"user","content":"Example"},{"role":"system","content":"Help"}]} + ]}`) + + _, baseline, err := ParseSourceWithSelections( + sourceBundle(reference.File, before), id, syncdomain.VariationModeCompletion, reference, + ) + require.NoError(t, err) + _, reordered, err := ParseSourceWithSelections( + sourceBundle(reference.File, after), id, syncdomain.VariationModeCompletion, reference, + ) + require.NoError(t, err) + + require.Len(t, baseline, 2) + require.Len(t, reordered, 2) + assert.NotEqual(t, baseline, reordered) +} + +func TestYAMLUnicodeColumnsAndFlowScalarsUseExactSpans(t *testing.T) { + source := []byte("étiquette: {prompt: ancien, enabled: true}\n") + id := syncdomain.VariationID("project", "config", "default") + reference := syncdomain.Reference{ + File: "agents.yaml", Format: "yaml", Select: `["étiquette"]`, + Map: map[string]string{"prompt": "prompt"}, + } + prompt := "nouveau" + + result, err := UpdateSource( + sourceBundle(reference.File, source), id, syncdomain.VariationModeAgent, reference, + syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: &prompt}, + ) + + require.NoError(t, err) + updated := applyConformanceEdits(t, source, directEdits(t, result)) + assert.Equal(t, "étiquette: {prompt: nouveau, enabled: true}\n", string(updated)) +} + +func TestStructuredValueFormsRoundTrip(t *testing.T) { + tests := []struct { + name string + format string + source string + path string + want any + next any + }{ + {name: "JSON escaped Unicode string", format: "json", source: `{"value":"Hello \u4e16\u754c"}`, path: "value", want: "Hello 世界", next: "Hello 🌍"}, + {name: "JSON number", format: "json", source: `{"value":1.25}`, path: "value", want: 1.25, next: 2.5}, + {name: "JSON boolean", format: "json", source: `{"value":true}`, path: "value", want: true, next: false}, + {name: "JSON object", format: "json", source: `{"value":{"enabled":true}}`, path: "value", want: map[string]any{"enabled": true}, next: map[string]any{"enabled": false}}, + {name: "YAML plain string", format: "yaml", source: "value: hello\n", path: "value", want: "hello", next: "updated"}, + {name: "YAML single quoted string", format: "yaml", source: "value: 'hello'\n", path: "value", want: "hello", next: "updated"}, + {name: "YAML double quoted string", format: "yaml", source: "value: \"hello\\nworld\"\n", path: "value", want: "hello\nworld", next: "updated\nworld"}, + {name: "YAML number", format: "yaml", source: "value: 1.25\n", path: "value", want: 1.25, next: 2.5}, + {name: "YAML boolean", format: "yaml", source: "value: true\n", path: "value", want: true, next: false}, + {name: "TOML basic string", format: "toml", source: "value = \"hello\"\n", path: "value", want: "hello", next: "updated"}, + {name: "TOML literal string", format: "toml", source: "value = 'hello'\n", path: "value", want: "hello", next: "updated"}, + {name: "TOML multiline string", format: "toml", source: "value = \"\"\"hello\nworld\"\"\"\n", path: "value", want: "hello\nworld", next: "updated\nworld"}, + {name: "TOML number", format: "toml", source: "value = 1.25\n", path: "value", want: 1.25, next: 2.5}, + {name: "TOML boolean", format: "toml", source: "value = true\n", path: "value", want: true, next: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source := []byte(test.source) + root, err := parseSourceDocument(Document{Path: "source", Data: source}, test.format) + require.NoError(t, err) + node, err := resolveSourcePath(root, test.path) + require.NoError(t, err) + assert.Equal(t, test.want, node.value) + require.Equal(t, CapabilityDirect, node.capability) + replacement, err := node.encode(test.next) + require.NoError(t, err) + updated := append(append(append([]byte{}, source[:node.start]...), replacement...), source[node.end:]...) + root, err = parseSourceDocument(Document{Path: "source", Data: updated}, test.format) + require.NoError(t, err) + node, err = resolveSourcePath(root, test.path) + require.NoError(t, err) + assert.Equal(t, test.next, node.value) + }) + } +} + +func TestStructuredProjectionCollectionsAndObjects(t *testing.T) { + variationSource := []byte(`{ + "prompt": "Help", + "tools": ["source-tool"], + "skills": ["source-skill"] +}`) + variationID := syncdomain.VariationID("project", "config", "default") + tools := []syncdomain.AttachmentRef{{Key: "lookup"}} + skills := []syncdomain.AttachmentRef{{Key: "refunds"}} + variationRef := syncdomain.Reference{ + File: "variation.json", Format: "json", Select: "$", + Map: map[string]string{"prompt": "prompt"}, + Tools: &tools, Skills: &skills, + } + projection, err := ParseSource( + sourceBundle(variationRef.File, variationSource), variationID, + syncdomain.VariationModeAgent, variationRef, + ) + require.NoError(t, err) + variation := projection.(syncdomain.VariationProjection) + assert.Equal(t, []string{"lookup"}, *variation.Tools) + assert.Equal(t, []string{"refunds"}, *variation.Skills) + + toolSource := []byte(`{ + "description": "Lookup", + "schema": {"type": "object"}, + "customParameters": {"timeout": 5}, + "tags": ["orders", "support"] +}`) + toolID := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "lookup"} + toolRef := syncdomain.Reference{ + File: "tool.json", Format: "json", Select: "$", + Map: map[string]string{ + "description": "description", "schema": "schema", + "customParameters": "customParameters", "tags": "tags", + }, + } + projection, err = ParseSource(sourceBundle(toolRef.File, toolSource), toolID, "", toolRef) + require.NoError(t, err) + tool := projection.(syncdomain.ToolProjection) + assert.Equal(t, "object", (*tool.Schema)["type"]) + assert.Equal(t, float64(5), (*tool.CustomParameters)["timeout"]) + assert.Equal(t, []string{"orders", "support"}, *tool.Tags) +} + +func TestMapTypeMismatchUsesStableCode(t *testing.T) { + id := syncdomain.VariationID("project", "config", "default") + reference := syncdomain.Reference{ + File: "value.json", Format: "json", Select: "$", + Map: map[string]string{"prompt": "prompt"}, + } + + _, err := ParseSource( + sourceBundle(reference.File, []byte(`{"prompt":42}`)), + id, syncdomain.VariationModeAgent, reference, + ) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorMapTypeMismatch, problem.Code) +} diff --git a/internal/sync/reference/testdata/json/mcp.json b/internal/sync/reference/testdata/json/mcp.json new file mode 100644 index 000000000..341469504 --- /dev/null +++ b/internal/sync/reference/testdata/json/mcp.json @@ -0,0 +1,32 @@ +{ + "server": "orders", + "tools": [ + { + "name": "lookup_order", + "description": "Look up one order.", + "inputSchema": { + "type": "object", + "properties": { + "orderId": { "type": "string" } + }, + "required": ["orderId"] + } + }, + { + "name": "refund_order", + "description": "Refund one order.", + "inputSchema": { + "type": "object", + "properties": { + "orderId": { "type": "string" }, + "reason": { "type": "string" } + }, + "required": ["orderId", "reason"] + } + } + ], + "metadata": { + "display.name": "Order tools", + "enabled": true + } +} diff --git a/internal/sync/reference/testdata/toml/pyproject.toml b/internal/sync/reference/testdata/toml/pyproject.toml new file mode 100644 index 000000000..689ed0c28 --- /dev/null +++ b/internal/sync/reference/testdata/toml/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name = "support-agents" +version = "1.0.0" + +[tool.agent.support] +prompt = """Ask for an order ID. +Keep the answer short.""" +model = "gpt-5" +temperature = 0.2 +enabled = true + +[tool.agent.refunds] +prompt = 'Explain the refund policy.' +model = 'gpt-5' +temperature = 0.1 +enabled = false + +[[tool.agent.plugins]] +name = "audit" +enabled = true diff --git a/internal/sync/reference/testdata/yaml/agents.yaml b/internal/sync/reference/testdata/yaml/agents.yaml new file mode 100644 index 000000000..2bb224b47 --- /dev/null +++ b/internal/sync/reference/testdata/yaml/agents.yaml @@ -0,0 +1,16 @@ +service: support +agents: + - name: support + prompt: "Ask for an order ID." + model: + name: gpt-5 + temperature: 0.2 + enabled: true + - name: refunds + prompt: 'Explain the refund policy.' + model: + name: gpt-5 + temperature: 0.1 + enabled: false +metadata: + "display.name": Support agents diff --git a/internal/sync/reference/toml.go b/internal/sync/reference/toml.go new file mode 100644 index 000000000..6bb342488 --- /dev/null +++ b/internal/sync/reference/toml.go @@ -0,0 +1,307 @@ +package reference + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "unicode/utf8" + + toml "github.com/pelletier/go-toml/v2" + "github.com/pelletier/go-toml/v2/unstable" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type tomlAdapter struct{} + +func (tomlAdapter) Format() string { return "toml" } +func (tomlAdapter) Inspect(bundle SourceBundle) (Inspection, error) { + root, err := parseTOMLDocument(bundle.Primary) + return Inspection{root: root}, err +} +func (tomlAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "$", nil } + +func init() { mustRegister(tomlAdapter{}) } + +func parseTOMLDocument(document Document) (*sourceNode, error) { + if !utf8.Valid(document.Data) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "TOML source is not valid UTF-8", + "Save the file as UTF-8.", + ).WithFile(document.Path, 0, 0) + } + data := document.Data + offset := 0 + if bytes.HasPrefix(data, []byte("\ufeff")) { + offset = len([]byte("\ufeff")) + data = data[offset:] + } + var parser unstable.Parser + parser.Reset(data) + root := &sourceNode{value: map[string]any{}, fields: map[string]*sourceNode{}} + table := root + for parser.NextExpression() { + expression := parser.Expression() + switch expression.Kind { + case unstable.KeyValue: + keys := tomlKeys(expression) + if err := addTOMLValue(data, offset, table, keys, expression.Value()); err != nil { + return nil, err + } + case unstable.Table: + var err error + table, err = tomlTable(root, tomlKeys(expression), false) + if err != nil { + return nil, err + } + case unstable.ArrayTable: + var err error + table, err = tomlTable(root, tomlKeys(expression), true) + if err != nil { + return nil, err + } + } + } + if err := parser.Error(); err != nil { + return nil, syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "TOML source is invalid", "Fix the TOML syntax.", + ).WithFile(document.Path, 0, 0), err) + } + return root, nil +} + +func tomlKeys(node *unstable.Node) []string { + var keys []string + iterator := node.Key() + for iterator.Next() { + keys = append(keys, string(iterator.Node().Data)) + } + return keys +} + +func addTOMLValue(data []byte, offset int, table *sourceNode, keys []string, valueNode *unstable.Node) error { + if len(keys) == 0 { + return syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "TOML key is missing", "Add a key to the value.", + ) + } + parent, err := tomlObjectPath(table, keys[:len(keys)-1]) + if err != nil { + return err + } + key := keys[len(keys)-1] + if _, duplicate := parent.fields[key]; duplicate { + return syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("TOML table contains duplicate key %q", strings.Join(keys, ".")), + "Remove the duplicate key.", + ) + } + value, err := tomlValueNode(data, offset, valueNode) + if err != nil { + return err + } + if table.capability == CapabilityReadOnly { + markReadOnly(value, table.reason) + } + parent.fields[key] = value + parent.value.(map[string]any)[key] = value.value + return nil +} + +func tomlTable(root *sourceNode, keys []string, array bool) (*sourceNode, error) { + if !array { + return tomlObjectPath(root, keys) + } + if len(keys) == 0 { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "TOML array table key is missing", "Add a table key.", + ) + } + parent, err := tomlObjectPath(root, keys[:len(keys)-1]) + if err != nil { + return nil, err + } + key := keys[len(keys)-1] + collection := parent.fields[key] + if collection == nil { + collection = &sourceNode{ + value: []any{}, capability: CapabilityReadOnly, + reason: "TOML arrays of tables are read-only", + } + parent.fields[key] = collection + parent.value.(map[string]any)[key] = collection.value + } + if collection.items == nil && len(collection.value.([]any)) != 0 { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("TOML key %q has incompatible definitions", strings.Join(keys, ".")), + "Keep one table definition.", + ) + } + item := &sourceNode{ + value: map[string]any{}, fields: map[string]*sourceNode{}, + capability: CapabilityReadOnly, reason: "TOML arrays of tables are read-only", + } + collection.items = append(collection.items, item) + collection.value = append(collection.value.([]any), item.value) + parent.value.(map[string]any)[key] = collection.value + return item, nil +} + +func tomlObjectPath(root *sourceNode, keys []string) (*sourceNode, error) { + current := root + for _, key := range keys { + child := current.fields[key] + if child == nil { + child = &sourceNode{value: map[string]any{}, fields: map[string]*sourceNode{}} + current.fields[key] = child + current.value.(map[string]any)[key] = child.value + } + if child.fields == nil { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("TOML key %q is already a value", key), + "Keep one compatible key definition.", + ) + } + current = child + } + return current, nil +} + +func tomlValueNode(data []byte, offset int, node *unstable.Node) (*sourceNode, error) { + start := offset + int(node.Raw.Offset) + end := start + int(node.Raw.Length) + raw := data[node.Raw.Offset : node.Raw.Offset+node.Raw.Length] + switch node.Kind { + case unstable.String: + return stringNode(string(node.Data), start, end, func(value string) ([]byte, error) { + return encodeTOMLString(value, string(raw)) + }), nil + case unstable.Bool: + value := string(node.Data) == "true" + return &sourceNode{ + value: value, start: start, end: end, capability: CapabilityDirect, + encode: func(next any) ([]byte, error) { + value, ok := next.(bool) + if !ok { + return nil, mapTypeError("source", "a boolean") + } + return []byte(strconv.FormatBool(value)), nil + }, + }, nil + case unstable.Integer, unstable.Float: + var decoded map[string]any + if err := toml.Unmarshal(append([]byte("value = "), raw...), &decoded); err != nil { + return nil, err + } + number, ok := sourceNumber(decoded["value"]) + if !ok { + return readOnlyNode(start, end, "TOML number is not representable"), nil + } + return numberNode(number, start, end, func(next float64) ([]byte, error) { + return []byte(strconv.FormatFloat(next, 'g', -1, 64)), nil + }), nil + case unstable.Array: + return tomlArrayNode(data, offset, node) + case unstable.InlineTable: + return tomlInlineTableNode(data, offset, node) + default: + result := readOnlyNode(start, end, "TOML date and time values are read-only") + result.value = string(node.Data) + return result, nil + } +} + +func tomlArrayNode(data []byte, offset int, node *unstable.Node) (*sourceNode, error) { + var items []*sourceNode + var values []any + iterator := node.Children() + for iterator.Next() { + child := iterator.Node() + if child.Kind == unstable.Comment { + continue + } + item, err := tomlValueNode(data, offset, child) + if err != nil { + return nil, err + } + items, values = append(items, item), append(values, item.value) + } + return &sourceNode{ + value: values, items: items, collection: true, capability: CapabilityReadOnly, + reason: "TOML structural changes are read-only", + }, nil +} + +func tomlInlineTableNode(data []byte, offset int, node *unstable.Node) (*sourceNode, error) { + result := &sourceNode{ + value: map[string]any{}, fields: map[string]*sourceNode{}, + capability: CapabilityDirect, + } + iterator := node.Children() + for iterator.Next() { + child := iterator.Node() + if child.Kind != unstable.KeyValue { + continue + } + if err := addTOMLValue(data, offset, result, tomlKeys(child), child.Value()); err != nil { + return nil, err + } + } + result.capability = CapabilityReadOnly + result.reason = "TOML structural changes are read-only" + return result, nil +} + +func encodeTOMLString(value, previous string) ([]byte, error) { + switch { + case strings.HasPrefix(previous, `"""`): + return []byte(`"""` + strings.ReplaceAll(value, `"""`, `\"""`) + `"""`), nil + case strings.HasPrefix(previous, `'''`): + if strings.Contains(value, `'''`) { + return nil, fmt.Errorf("multiline literal string cannot contain three apostrophes") + } + return []byte(`'''` + value + `'''`), nil + case strings.HasPrefix(previous, `'`): + if strings.ContainsAny(value, "'\r\n") { + return nil, fmt.Errorf("literal string cannot represent the new value") + } + return []byte("'" + value + "'"), nil + default: + return []byte(encodeTOMLBasicString(value)), nil + } +} + +func encodeTOMLBasicString(value string) string { + var encoded strings.Builder + encoded.WriteByte('"') + for _, character := range value { + switch character { + case '"', '\\': + encoded.WriteByte('\\') + encoded.WriteRune(character) + case '\b': + encoded.WriteString(`\b`) + case '\t': + encoded.WriteString(`\t`) + case '\n': + encoded.WriteString(`\n`) + case '\f': + encoded.WriteString(`\f`) + case '\r': + encoded.WriteString(`\r`) + default: + if character < 0x20 || character >= 0x7f && character <= 0x9f { + fmt.Fprintf(&encoded, `\u%04X`, character) + } else { + encoded.WriteRune(character) + } + } + } + encoded.WriteByte('"') + return encoded.String() +} diff --git a/internal/sync/reference/yaml.go b/internal/sync/reference/yaml.go new file mode 100644 index 000000000..72253f579 --- /dev/null +++ b/internal/sync/reference/yaml.go @@ -0,0 +1,218 @@ +package reference + +import ( + "bytes" + "fmt" + "io" + "strconv" + "strings" + "unicode/utf8" + + "gopkg.in/yaml.v3" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type yamlAdapter struct{} + +func (yamlAdapter) Format() string { return "yaml" } +func (yamlAdapter) Inspect(bundle SourceBundle) (Inspection, error) { + root, err := parseYAMLDocument(bundle.Primary) + return Inspection{root: root}, err +} +func (yamlAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "$", nil } + +func init() { mustRegister(yamlAdapter{}) } + +func parseYAMLDocument(document Document) (*sourceNode, error) { + if !utf8.Valid(document.Data) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "YAML source is not valid UTF-8", + "Save the file as UTF-8.", + ).WithFile(document.Path, 0, 0) + } + data := bytes.TrimPrefix(document.Data, []byte("\ufeff")) + offset := len(document.Data) - len(data) + decoder := yaml.NewDecoder(bytes.NewReader(data)) + var first yaml.Node + if err := decoder.Decode(&first); err != nil { + return nil, syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML source is invalid", "Fix the YAML syntax.", + ).WithFile(document.Path, 0, 0), err) + } + var extra yaml.Node + extraErr := decoder.Decode(&extra) + multiple := extraErr == nil + if extraErr != nil && extraErr != io.EOF { + return nil, syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML source is invalid", "Fix the YAML syntax.", + ).WithFile(document.Path, 0, 0), extraErr) + } + if len(first.Content) == 0 { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML document is empty", "Add one YAML value.", + ) + } + root, err := yamlSourceNode(data, first.Content[0], offset) + if err != nil { + return nil, err + } + if multiple { + markReadOnly(root, "multi-document YAML is read-only") + } + return root, nil +} + +func yamlSourceNode(data []byte, node *yaml.Node, offset int) (*sourceNode, error) { + switch node.Kind { + case yaml.MappingNode: + return yamlMappingNode(data, node, offset) + case yaml.SequenceNode: + return yamlSequenceNode(data, node, offset) + case yaml.ScalarNode: + return yamlValueNode(data, node, offset) + case yaml.AliasNode: + value, err := yamlSourceNode(data, node.Alias, offset) + if value != nil { + markReadOnly(value, "YAML aliases are read-only") + } + return value, err + default: + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML node is not supported", "Use mappings, lists, or scalar values.", + ) + } +} + +func yamlMappingNode(data []byte, node *yaml.Node, offset int) (*sourceNode, error) { + fields := make(map[string]*sourceNode, len(node.Content)/2) + value := make(map[string]any, len(node.Content)/2) + unsafe := node.Anchor != "" || customYAMLTag(node.Tag) + reason := "YAML anchors and tags are read-only" + for index := 0; index+1 < len(node.Content); index += 2 { + keyNode, valueNode := node.Content[index], node.Content[index+1] + if keyNode.Kind != yaml.ScalarNode || keyNode.Tag != "!!str" && keyNode.Tag != "!!merge" { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "YAML mapping key is not a string", + "Use direct string keys.", + ) + } + key := keyNode.Value + if _, duplicate := fields[key]; duplicate { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("YAML mapping contains duplicate key %q", key), + "Remove the duplicate key.", + ) + } + child, err := yamlSourceNode(data, valueNode, offset) + if err != nil { + return nil, err + } + fields[key], value[key] = child, child.value + if key == "<<" { + unsafe, reason = true, "YAML merge keys are read-only" + for mergedKey, mergedValue := range child.fields { + if _, exists := fields[mergedKey]; !exists { + fields[mergedKey] = mergedValue + value[mergedKey] = mergedValue.value + } + } + } + } + result := &sourceNode{ + value: value, fields: fields, capability: CapabilityReadOnly, + reason: "YAML structural changes are read-only", + } + if unsafe { + markReadOnly(result, reason) + } + return result, nil +} + +func yamlSequenceNode(data []byte, node *yaml.Node, offset int) (*sourceNode, error) { + items := make([]*sourceNode, 0, len(node.Content)) + values := make([]any, 0, len(node.Content)) + for _, childNode := range node.Content { + child, err := yamlSourceNode(data, childNode, offset) + if err != nil { + return nil, err + } + items, values = append(items, child), append(values, child.value) + } + result := &sourceNode{ + value: values, items: items, collection: true, capability: CapabilityReadOnly, + reason: "YAML structural changes are read-only", + } + if node.Anchor != "" || customYAMLTag(node.Tag) { + markReadOnly(result, "YAML anchors and tags are read-only") + } + return result, nil +} + +func yamlValueNode(data []byte, node *yaml.Node, offset int) (*sourceNode, error) { + region := frontMatterRegion{data: data, startOffset: offset} + start, end, raw, err := yamlScalarSpan(region, node) + if err != nil { + return nil, err + } + readOnly := node.Anchor != "" || customYAMLTag(node.Tag) || + node.Style == yaml.LiteralStyle || node.Style == yaml.FoldedStyle + reason := "YAML anchors, tags, and block scalars are read-only" + var result *sourceNode + switch node.Tag { + case "!!str": + result = stringNode(node.Value, start, end, func(text string) ([]byte, error) { + return encodeYAMLString(text, raw) + }) + case "!!bool": + value, err := strconv.ParseBool(node.Value) + if err != nil { + return nil, err + } + result = &sourceNode{ + value: value, start: start, end: end, capability: CapabilityDirect, + encode: func(next any) ([]byte, error) { + value, ok := next.(bool) + if !ok { + return nil, mapTypeError("source", "a boolean") + } + return []byte(strconv.FormatBool(value)), nil + }, + } + case "!!int", "!!float": + number, err := strconv.ParseFloat(strings.ReplaceAll(node.Value, "_", ""), 64) + if err != nil { + return nil, err + } + result = numberNode(number, start, end, func(next float64) ([]byte, error) { + return []byte(strconv.FormatFloat(next, 'g', -1, 64)), nil + }) + default: + result = readOnlyNode(start, end, "YAML scalar type is read-only") + result.value = node.Value + } + if readOnly { + result.capability, result.reason, result.encode = CapabilityReadOnly, reason, nil + } + return result, nil +} + +func customYAMLTag(tag string) bool { + return tag != "" && !strings.HasPrefix(tag, "!!") +} + +func markReadOnly(node *sourceNode, reason string) { + if node == nil { + return + } + node.capability, node.reason, node.encode = CapabilityReadOnly, reason, nil + for _, child := range node.fields { + markReadOnly(child, reason) + } + for _, child := range node.items { + markReadOnly(child, reason) + } +} From 27de43a78ed8146671537abf0e0a046eeaaa0b75 Mon Sep 17 00:00:00 2001 From: Cliff Tawiah Date: Fri, 30 Oct 2026 16:58:36 -0400 Subject: [PATCH 5/5] feat(sync): add provider profile adapters --- internal/sync/reference/defensive_test.go | 8 ++ internal/sync/reference/profile_test.go | 81 +++++++++++ internal/sync/reference/profiles.go | 126 ++++++++++++++++++ .../profiles/agents-instructions.md.golden | 13 ++ .../profiles/claude-instructions.md.golden | 10 ++ .../testdata/profiles/copilot.instructions.md | 7 + .../testdata/profiles/cursor-rule.mdc | 9 ++ 7 files changed, 254 insertions(+) create mode 100644 internal/sync/reference/profile_test.go create mode 100644 internal/sync/reference/profiles.go create mode 100644 internal/sync/reference/testdata/profiles/agents-instructions.md.golden create mode 100644 internal/sync/reference/testdata/profiles/claude-instructions.md.golden create mode 100644 internal/sync/reference/testdata/profiles/copilot.instructions.md create mode 100644 internal/sync/reference/testdata/profiles/cursor-rule.mdc diff --git a/internal/sync/reference/defensive_test.go b/internal/sync/reference/defensive_test.go index 064511af6..8ae4bcafc 100644 --- a/internal/sync/reference/defensive_test.go +++ b/internal/sync/reference/defensive_test.go @@ -33,3 +33,11 @@ func TestStructuredAdaptersRejectMalformedSources(t *testing.T) { }) } } + +func TestMarkdownProfilesRejectUnsafeMarkers(t *testing.T) { + _, err := parseMarkdownProfile( + Document{Path: "instructions.md", Data: []byte("\n")}, + false, + ) + require.Error(t, err) +} diff --git a/internal/sync/reference/profile_test.go b/internal/sync/reference/profile_test.go new file mode 100644 index 000000000..734af8350 --- /dev/null +++ b/internal/sync/reference/profile_test.go @@ -0,0 +1,81 @@ +package reference + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestMarkdownProfilesPreserveActivationMetadata(t *testing.T) { + tests := []struct { + name, fixture, realPath, format, metadataPath string + metadataValue any + }{ + {name: "Cursor", fixture: "profiles/cursor-rule.mdc", realPath: ".cursor/rules/support.mdc", format: "cursor", metadataPath: "frontmatter.globs", metadataValue: "src/support/**/*.ts"}, + {name: "Copilot", fixture: "profiles/copilot.instructions.md", realPath: ".github/instructions/support.instructions.md", format: "copilot", metadataPath: "frontmatter.applyTo", metadataValue: "src/support/**/*.ts"}, + {name: "Claude", fixture: "profiles/claude-instructions.md.golden", realPath: "CLAUDE.md", format: "claude-markdown", metadataPath: "frontmatter.model", metadataValue: "sonnet"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + documentPath, source := materializeFixture(t, test.fixture, test.realPath) + id := syncdomain.VariationID("project", "config", "default") + reference := syncdomain.Reference{File: documentPath, Format: test.format} + root, err := parseSourceDocument(Document{Path: documentPath, Data: source}, test.format) + require.NoError(t, err) + metadata, err := resolveSourcePath(root, test.metadataPath) + require.NoError(t, err) + assert.Equal(t, test.metadataValue, metadata.value) + + projection, err := ParseSource(sourceBundle(documentPath, source), id, syncdomain.VariationModeAgent, reference) + require.NoError(t, err) + prompt := "Updated profile instructions." + result, err := UpdateSource( + sourceBundle(documentPath, source), id, syncdomain.VariationModeAgent, reference, + syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: &prompt}, + ) + require.NoError(t, err) + updated := applyConformanceEdits(t, source, directEdits(t, result)) + root, err = parseSourceDocument(Document{Path: documentPath, Data: updated}, test.format) + require.NoError(t, err) + metadata, err = resolveSourcePath(root, test.metadataPath) + require.NoError(t, err) + assert.Equal(t, test.metadataValue, metadata.value) + assert.NotNil(t, projection) + }) + } +} + +func TestAgentsMarkdownSectionsAndMarkers(t *testing.T) { + documentPath, source := materializeFixture(t, "profiles/agents-instructions.md.golden", "AGENTS.md") + id := syncdomain.VariationID("project", "config", "default") + tests := []struct { + name, selectPath, mapPath, want string + }{ + {name: "section", selectPath: `sections[heading="Support"]`, mapPath: "body", want: "Ask for an order ID.\n\n\nExplain the refund policy.\n"}, + {name: "marker", selectPath: `markers[id="refunds"]`, mapPath: "body", want: "Explain the refund policy."}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + reference := syncdomain.Reference{ + File: documentPath, Format: "agents-markdown", Select: test.selectPath, + Map: map[string]string{"prompt": test.mapPath}, + } + projection, err := ParseSource( + sourceBundle(reference.File, source), id, syncdomain.VariationModeAgent, reference, + ) + require.NoError(t, err) + assert.Equal(t, test.want, *projection.(syncdomain.VariationProjection).Instructions) + prompt := "Updated selected instructions." + result, err := UpdateSource( + sourceBundle(reference.File, source), id, syncdomain.VariationModeAgent, reference, + syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: &prompt}, + ) + require.NoError(t, err) + updated := applyConformanceEdits(t, source, directEdits(t, result)) + assert.Contains(t, string(updated), "## Build\n\nRun `make test`.") + }) + } +} diff --git a/internal/sync/reference/profiles.go b/internal/sync/reference/profiles.go new file mode 100644 index 000000000..68aeb103d --- /dev/null +++ b/internal/sync/reference/profiles.go @@ -0,0 +1,126 @@ +package reference + +import ( + "bytes" + "fmt" + "strings" + + "gopkg.in/yaml.v3" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type markdownProfileAdapter struct { + format string + frontMatter bool +} + +func (adapter markdownProfileAdapter) Format() string { return adapter.format } +func (adapter markdownProfileAdapter) Inspect(bundle SourceBundle) (Inspection, error) { + root, err := parseMarkdownProfile(bundle.Primary, adapter.frontMatter) + return Inspection{root: root}, err +} +func (markdownProfileAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + if kind == syncdomain.KindVariation { + return "$", map[string]string{string(syncdomain.FieldPrompt): "body"} + } + return "$", nil +} + +func init() { + mustRegister(markdownProfileAdapter{format: "cursor", frontMatter: true}) + mustRegister(markdownProfileAdapter{format: "copilot", frontMatter: true}) + mustRegister(markdownProfileAdapter{format: "agents-markdown"}) + mustRegister(markdownProfileAdapter{format: "claude-markdown", frontMatter: true}) +} + +func parseMarkdownProfile(document Document, supportsFrontMatter bool) (*sourceNode, error) { + data := document.Data + bodyStart := 0 + frontmatter := &sourceNode{value: map[string]any{}, fields: map[string]*sourceNode{}} + trimmed := bytes.TrimPrefix(data, []byte("\ufeff")) + bomLength := len(data) - len(trimmed) + if supportsFrontMatter && bytes.HasPrefix(trimmed, []byte("---")) { + region, start, err := splitFrontMatter(data) + if err != nil { + return nil, err + } + bodyStart = start + var documentNode yaml.Node + if err := yaml.Unmarshal(region.data, &documentNode); err != nil { + return nil, fmt.Errorf("invalid profile front matter: %w", err) + } + if len(documentNode.Content) != 0 { + frontmatter, err = yamlSourceNode(region.data, documentNode.Content[0], region.startOffset) + if err != nil { + return nil, err + } + } + } else { + bodyStart = bomLength + } + bodyData := data[bodyStart:] + body := markdownTextNode(bodyData, bodyStart, len(data)) + sections := &sourceNode{items: markdownSections(bodyData), collection: true} + offsetSourceSpans(sections, bodyStart) + markers, err := markdownMarkers(bodyData, bodyStart) + if err != nil { + return nil, err + } + return &sourceNode{fields: map[string]*sourceNode{ + "frontmatter": frontmatter, + "body": body, + "sections": sections, + "markers": markers, + }}, nil +} + +func markdownMarkers(data []byte, base int) (*sourceNode, error) { + const beginPrefix = ""): + if openID != "" { + return nil, fmt.Errorf("nested LaunchDarkly marker %q", openID) + } + openID = strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(line, beginPrefix), "-->")) + bodyStart = lineEnd + case strings.HasPrefix(line, endPrefix) && strings.HasSuffix(line, "-->"): + id := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(line, endPrefix), "-->")) + if openID == "" || id != openID { + return nil, fmt.Errorf("LaunchDarkly marker %q has no matching begin marker", id) + } + body := markdownTextNode(data[bodyStart:offset], base+bodyStart, base+offset) + items = append(items, &sourceNode{fields: map[string]*sourceNode{ + "id": {value: id}, "body": body, + }}) + openID = "" + } + offset = lineEnd + } + if openID != "" { + return nil, fmt.Errorf("LaunchDarkly marker %q is not closed", openID) + } + return &sourceNode{items: items, collection: true}, nil +} + +func offsetSourceSpans(node *sourceNode, offset int) { + if node == nil { + return + } + if node.start != 0 || node.end != 0 { + node.start += offset + node.end += offset + } + for _, child := range node.fields { + offsetSourceSpans(child, offset) + } + for _, child := range node.items { + offsetSourceSpans(child, offset) + } +} diff --git a/internal/sync/reference/testdata/profiles/agents-instructions.md.golden b/internal/sync/reference/testdata/profiles/agents-instructions.md.golden new file mode 100644 index 000000000..28d9f4674 --- /dev/null +++ b/internal/sync/reference/testdata/profiles/agents-instructions.md.golden @@ -0,0 +1,13 @@ +# Repository guide + +## Support + +Ask for an order ID. + + +Explain the refund policy. + + +## Build + +Run `make test`. diff --git a/internal/sync/reference/testdata/profiles/claude-instructions.md.golden b/internal/sync/reference/testdata/profiles/claude-instructions.md.golden new file mode 100644 index 000000000..f4a10263c --- /dev/null +++ b/internal/sync/reference/testdata/profiles/claude-instructions.md.golden @@ -0,0 +1,10 @@ +--- +name: support +description: Support subagent instructions +tools: Read, Grep +model: sonnet +--- + +# Support subagent + +Ask for an order ID and follow the support policy. diff --git a/internal/sync/reference/testdata/profiles/copilot.instructions.md b/internal/sync/reference/testdata/profiles/copilot.instructions.md new file mode 100644 index 000000000..024f8f369 --- /dev/null +++ b/internal/sync/reference/testdata/profiles/copilot.instructions.md @@ -0,0 +1,7 @@ +--- +applyTo: "src/support/**/*.ts" +--- + +# Support + +Use the support policy. diff --git a/internal/sync/reference/testdata/profiles/cursor-rule.mdc b/internal/sync/reference/testdata/profiles/cursor-rule.mdc new file mode 100644 index 000000000..6eef5da26 --- /dev/null +++ b/internal/sync/reference/testdata/profiles/cursor-rule.mdc @@ -0,0 +1,9 @@ +--- +description: Support instructions +globs: "src/support/**/*.ts" +alwaysApply: false +--- + +# Support + +Ask for an order ID before discussing an order.