feat(sync): add source descriptor storage

internal/sync/detach/detach_test.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+	"syscall"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -425,11 +426,11 @@ func TestRunKeepsEveryFileWhenOneDeleteFails(t *testing.T) {
 	workspace := newArchiveWorkspace(t, "prompt")
 	tool := testTool()
 	workspace.addAttachments(t, tool)
-	// A symbolic link in place of the tool file makes its delete fail.
+	// A named pipe in place of the tool file is not a regular file, so its
+	// delete fails.
 	toolFile := filepath.Join(workspace.root, syncdomain.RootDir, "project", "tools", "search.json")
-	target := filepath.Join(t.TempDir(), "search.json")
-	require.NoError(t, os.Rename(toolFile, target))
-	require.NoError(t, os.Symlink(target, toolFile))
+	require.NoError(t, os.Remove(toolFile))
+	require.NoError(t, syscall.Mkfifo(toolFile, 0o644))
 	options := workspace.options(&recordingArchiver{})
 	options.Selections = append(options.Selections, tool.ID("project"))
 	options.Yes = true

internal/sync/local/create.go

@@ -4,8 +4,11 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
+
+	syncdomain "github.com/launchdarkly/ldcli/internal/sync"
 )
 
 // Creation records the result of a create. VariationPaths are the variation
@@ -73,17 +76,54 @@ func (store Store) Bootstrap(files []VariationFile) (Creation, error) {
 }
 
 // Add creates a batch of variation files and the tool and skill files that
-// they use. Add never overwrites a file. An existing variation file is an
-// error, and an existing tool or skill file stays as it is, so that a local
-// edit is not lost. If one file fails, Add removes the files that it created.
+// they use. Add never overwrites a file. An existing variation file or
+// descriptor is an error, and an existing tool or skill, as a managed file or
+// as a descriptor, stays as it is, so that a local edit is not lost. If one
+// file fails, Add removes the files that it created.
 func (store Store) Add(files []VariationFile) (Creation, error) {
 	rendered, err := store.Render(files)
 	if err != nil {
 		return Creation{}, err
 	}
+	linked := make(map[string]bool)
+	for _, file := range rendered {
+		id, ok := ParseManagedPath(path.Join(syncdomain.RootDir, file.Path))
+		if !ok {
+			continue
+		}
+		repositoryReferencePath, err := ReferencePath(id)
+		if err != nil {
+			return Creation{}, err
+		}
+		referencePath := strings.TrimPrefix(repositoryReferencePath, syncdomain.RootDir+"/")
+		exists, err := store.fileExists(referencePath, id.Label()+" descriptor")
+		if err != nil {
+			return Creation{}, err
+		}
+		if !exists {
+			continue
+		}
+		if id.Kind == syncdomain.KindVariation {
+			return Creation{}, definitionConflict(id, id.Label()+" already has a descriptor")
+		}
+		// A tool or skill that a descriptor defines has that one definition.
+		// A managed file beside the descriptor is a conflict, not a file to
+		// keep.
+		managed, err := store.fileExists(file.Path, id.Label())
+		if err != nil {
+			return Creation{}, err
+		}
+		if managed {
+			return Creation{}, definitionConflict(id, id.Label()+" has a managed file and a descriptor")
+		}
+		linked[file.Path] = true
+	}
 
 	var creation Creation
 	for _, file := range rendered {
+		if linked[file.Path] {
+			continue
+		}
 		created, err := createFileIfAbsent(store.root, store.absolute(file.Path), file.Content)
 		if err != nil {
 			return Creation{}, errors.Join(err, store.RollbackCreation(creation))
@@ -116,8 +156,18 @@ func (store Store) RollbackCreation(creation Creation) error {
 // createFileIfAbsent creates one file and reports whether it did. An existing
 // variation file is an error. An existing tool or skill file is kept.
 func createFileIfAbsent(root, path string, data []byte) (bool, error) {
-	_, err := os.Stat(path)
+	info, err := os.Lstat(path)
 	switch {
+	case err == nil && info.Mode()&os.ModeSymlink != 0:
+		// A leftover link at a managed path is not a definition, as it is
+		// not for compile, so the add replaces it with the file. The
+		// parents of the path must not be links.
+		if err := rejectSymlinkedPath(root, filepath.Dir(path)); err != nil {
+			return false, err
+		}
+		if err := os.Remove(path); err != nil {
+			return false, fmt.Errorf("remove leftover link %s: %w", filepath.Base(path), err)
+		}
 	case err == nil:
 		if err := rejectSymlinkedPath(root, path); err != nil {
 			return false, err

internal/sync/local/delete.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	syncdomain "github.com/launchdarkly/ldcli/internal/sync"
 )
@@ -44,6 +45,69 @@ func (store Store) DeleteResources(ids []syncdomain.ResourceID) ([]string, error
 	return store.deleteFiles(paths)
 }
 
+// DeleteDefinitions deletes each descriptor when present, or its managed file.
+// It never deletes the external file that a descriptor references.
+func (store Store) DeleteDefinitions(ids []syncdomain.ResourceID) ([]string, error) {
+	paths := make([]string, 0, len(ids))
+	for _, id := range ids {
+		reference, err := ReferencePath(id)
+		if err != nil {
+			return nil, err
+		}
+		reference = strings.TrimPrefix(reference, syncdomain.RootDir+"/")
+		referenced, err := store.fileExists(reference, id.Label()+" descriptor")
+		if err != nil {
+			return nil, err
+		}
+		managed, err := resourcePath(id)
+		if err != nil {
+			return nil, err
+		}
+		// A leftover link at the managed path is not a definition, as it is
+		// not for compile, so the descriptor beside it is the one to delete.
+		managedExists, err := store.fileExists(managed, id.Label())
+		if err != nil {
+			return nil, err
+		}
+		link, err := store.isLink(managed)
+		if err != nil {
+			return nil, err
+		}
+		switch {
+		case referenced && managedExists:
+			return nil, definitionConflict(id, id.Label()+" has a managed file and a descriptor")
+		case referenced:
+			paths = append(paths, reference)
+			// A leftover link beside the descriptor goes with it, in the
+			// same transaction, so that the link does not stand alone, where
+			// compile rejects it.
+			if link {
+				paths = append(paths, managed)
+			}
+		case managedExists, link:
+			// A lone leftover link goes too: sync owns the managed
+			// directory, and the delete removes the link, not its target.
+			paths = append(paths, managed)
+		default:
+			return nil, fmt.Errorf("inspect definition %s: %w", id.Label(), os.ErrNotExist)
+		}
+	}
+	return store.deleteFiles(paths)
+}
+
+// isLink reports whether a managed path is a symbolic link.
+func (store Store) isLink(relativePath string) (bool, error) {
+	info, err := os.Lstat(store.absolute(relativePath))
+	switch {
+	case err == nil:
+		return info.Mode()&os.ModeSymlink != 0, nil
+	case errors.Is(err, os.ErrNotExist):
+		return false, nil
+	default:
+		return false, fmt.Errorf("inspect %s: %w", relativePath, err)
+	}
+}
+
 // deleteFiles deletes a batch of managed files in one transaction. It first
 // moves every file to a backup beside it. If a move fails, it moves the
 // files back. When every file has moved, it removes the backups.
@@ -57,16 +121,24 @@ func (store Store) deleteFiles(relativePaths []string) ([]string, error) {
 		seen[relativePath] = struct{}{}
 
 		path := store.absolute(relativePath)
-		if err := rejectSymlinkedPath(store.root, path); err != nil {
-			return nil, err
-		}
-		// Lstat does not follow a symbolic link, so a link is not a regular file.
+		// Lstat does not follow a symbolic link, so a link is not a regular
+		// file. A link itself can go, as a rename moves the link and not its
+		// target, but no parent of a managed file is a link.
 		info, err := os.Lstat(path)
 		if err != nil {
 			return nil, fmt.Errorf("inspect %s: %w", relativePath, err)
 		}
-		if !info.Mode().IsRegular() {
-			return nil, fmt.Errorf("%s is not a regular file", relativePath)
+		if info.Mode()&os.ModeSymlink != 0 {
+			if err := rejectSymlinkedPath(store.root, filepath.Dir(path)); err != nil {
+				return nil, err
+			}
+		} else {
+			if err := rejectSymlinkedPath(store.root, path); err != nil {
+				return nil, err
+			}
+			if !info.Mode().IsRegular() {
+				return nil, fmt.Errorf("%s is not a regular file", relativePath)
+			}
 		}
 		deletions = append(deletions, stagedDeletion{relativePath: relativePath, path: path})
 	}

internal/sync/local/generated.go

@@ -0,0 +1,72 @@
+package local
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	syncdomain "github.com/launchdarkly/ldcli/internal/sync"
+)
+
+// This file tells an authored source from a generated, vendored, or minified
+// one, which sync does not edit.
+
+// publicationProblem is the problem of a source that sync cannot write.
+func publicationProblem(file, message, suggestion string, cause error) error {
+	return syncdomain.WrapProblem(
+		syncdomain.NewProblem(syncdomain.ErrorSourceNotWritable, message, suggestion).WithFile(file, 0, 0),
+		cause,
+	)
+}
+
+// generatedSource reports whether a path or its content marks a file that a
+// tool produced: a vendored or minified path, or a generated-code header.
+func generatedSource(file string, data []byte) bool {
+	// A case-insensitive volume serves `Vendor/` and `.Min.js` too, as the
+	// managed-directory check allows for.
+	path := strings.ToLower(filepath.ToSlash(file))
+	if strings.Contains("/"+path+"/", "/vendor/") || strings.HasSuffix(path, ".min.js") {
+		return true
+	}
+	prefix := strings.ToLower(string(data))
+	if len(prefix) > 512 {
+		prefix = prefix[:512]
+	}
+	return strings.Contains(prefix, "code generated") && strings.Contains(prefix, "do not edit")
+}
+
+// generatedTarget reports whether a source whose resolved path is target is
+// generated, vendored, or minified. A link can point into such a tree, so the
+// resolved path counts as well as the name in the descriptor, on every read.
+func generatedTarget(repositoryRoot, file, target string, data []byte) (bool, error) {
+	// A root or a target that does not resolve hides whether the source is
+	// writable, so the check fails closed.
+	root, err := filepath.EvalSymlinks(repositoryRoot)
+	if err != nil {
+		return false, fmt.Errorf("resolve repository root: %w", err)
+	}
+	resolved, err := filepath.Rel(root, target)
+	if err != nil {
+		return false, fmt.Errorf("resolve source file %q: %w", file, err)
+	}
+	return generatedSource(file, data) || generatedSource(filepath.ToSlash(resolved), data), nil
+}
+
+// rejectGeneratedTarget reports a source that sync must write but that is
+// generated, vendored, or minified.
+func rejectGeneratedTarget(repositoryRoot, file, target string, data []byte) error {
+	generated, err := generatedTarget(repositoryRoot, file, target, data)
+	if err != nil {
+		return err
+	}
+	if generated {
+		return generatedProblem(file)
+	}
+	return nil
+}
+
+// generatedProblem is the problem of a source that sync must write but that
+// a tool produced.
+func generatedProblem(file string) error {
+	return publicationProblem(file, "source is generated, vendored, or minified", "Link the authored source file.", nil)
+}

internal/sync/local/layout.go

@@ -26,6 +26,7 @@ const (
 	variationFileSuffix = ".prompt.md"
 	toolFileSuffix      = ".json"
 	skillFileSuffix     = ".md"
+	referenceFileSuffix = ".ref.yaml"
 )
 
 // ParseManagedPath returns the resource that a managed file stores. The path
@@ -62,6 +63,37 @@ func ParseManagedPath(file string) (syncdomain.ResourceID, bool) {
 	return id, true
 }
 
+// ParseReferencePath returns the resource that a .ref.yaml descriptor links.
+func ParseReferencePath(file string) (syncdomain.ResourceID, bool) {
+	parts := strings.Split(file, "/")
+	if len(parts) < 4 || parts[0] != syncdomain.RootDir {
+		return syncdomain.ResourceID{}, false
+	}
+
+	var id syncdomain.ResourceID
+	var stem string
+	switch {
+	case len(parts) == 5 && parts[2] == configsDir && strings.HasSuffix(parts[4], referenceFileSuffix):
+		stem = strings.TrimSuffix(parts[4], referenceFileSuffix)
+		if syncdomain.ValidateKey(parts[3]) != nil {
+			return syncdomain.ResourceID{}, false
+		}
+		id = syncdomain.VariationID(parts[1], parts[3], stem)
+	case len(parts) == 4 && parts[2] == toolsDir && strings.HasSuffix(parts[3], referenceFileSuffix):
+		stem = strings.TrimSuffix(parts[3], referenceFileSuffix)
+		id = syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: parts[1], LookupKey: stem}
+	case len(parts) == 4 && parts[2] == skillsDir && strings.HasSuffix(parts[3], referenceFileSuffix):
+		stem = strings.TrimSuffix(parts[3], referenceFileSuffix)
+		id = syncdomain.ResourceID{Kind: syncdomain.KindSkill, ProjectKey: parts[1], LookupKey: stem}
+	default:
+		return syncdomain.ResourceID{}, false
+	}
+	if syncdomain.ValidateKey(parts[1]) != nil || syncdomain.ValidateKey(stem) != nil {
+		return syncdomain.ResourceID{}, false
+	}
+	return id, true
+}
+
 // ResourcePath returns the path of the file of a variation, a tool, or a
 // skill relative to the repository root, in the form that ParseManagedPath
 // reads.
@@ -73,6 +105,34 @@ func ResourcePath(id syncdomain.ResourceID) (string, error) {
 	return path.Join(syncdomain.RootDir, relativePath), nil
 }
 
+// ReferencePath returns the repository-relative path of a resource descriptor.
+func ReferencePath(id syncdomain.ResourceID) (string, error) {
+	if err := validateKey("project", id.ProjectKey); err != nil {
+		return "", err
+	}
+	if id.Kind == syncdomain.KindVariation {
+		configKey, variationKey, err := id.VariationKeys()
+		if err != nil {
+			return "", err
+		}
+		if err := cmp.Or(validateKey("config", configKey), validateKey("variation", variationKey)); err != nil {
+			return "", err
+		}
+		return path.Join(syncdomain.RootDir, id.ProjectKey, configsDir, configKey, variationKey+referenceFileSuffix), nil
+	}
+	if err := validateKey(string(id.Kind), id.LookupKey); err != nil {
+		return "", err
+	}
+	switch id.Kind {
+	case syncdomain.KindTool:
+		return path.Join(syncdomain.RootDir, id.ProjectKey, toolsDir, id.LookupKey+referenceFileSuffix), nil
+	case syncdomain.KindSkill:
+		return path.Join(syncdomain.RootDir, id.ProjectKey, skillsDir, id.LookupKey+referenceFileSuffix), nil
+	default:
+		return "", fmt.Errorf("unsupported resource kind %q", id.Kind)
+	}
+}
+
 // variationPath returns the path of a variation file relative to the managed
 // directory. It makes sure that each key is a safe path segment.
 func variationPath(projectKey, configKey, variationKey string) (string, error) {