From aaaccc66470f717758df7ea4e16d8e39e75d02d6 Mon Sep 17 00:00:00 2001 From: Clifford Tawiah Date: Mon, 9 Nov 2026 13:44:56 -0500 Subject: [PATCH] feat(sync): add the Python adapter Add the shared tree-sitter helpers, string codecs, and placeholder translation. Add the Python adapter: it finds agents, prompts, tools, and constants through an import allowlist, follows static repository imports, and never runs Python. Co-Authored-By: Claude Fable 5.1 --- go.mod | 5 + go.sum | 30 + internal/sync/adapter/adapter.go | 9 +- internal/sync/adapter/code/code.go | 738 +++++ internal/sync/adapter/code/code_test.go | 210 ++ internal/sync/adapter/code/placeholder.go | 214 ++ internal/sync/adapter/python/frameworks.go | 301 +++ internal/sync/adapter/python/python.go | 2217 +++++++++++++++ internal/sync/adapter/python/python_test.go | 2694 +++++++++++++++++++ internal/sync/adapter/resolve.go | 47 +- 10 files changed, 6461 insertions(+), 4 deletions(-) create mode 100644 internal/sync/adapter/code/code.go create mode 100644 internal/sync/adapter/code/code_test.go create mode 100644 internal/sync/adapter/code/placeholder.go create mode 100644 internal/sync/adapter/python/frameworks.go create mode 100644 internal/sync/adapter/python/python.go create mode 100644 internal/sync/adapter/python/python_test.go diff --git a/go.mod b/go.mod index 83ca7bdef..4f7f646c4 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,10 @@ require ( github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.12.1 + github.com/tree-sitter/go-tree-sitter v0.25.0 + github.com/tree-sitter/tree-sitter-javascript v0.23.1 + github.com/tree-sitter/tree-sitter-python v0.25.0 + github.com/tree-sitter/tree-sitter-typescript v0.23.2 go.uber.org/mock v0.6.0 golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa golang.org/x/term v0.40.0 @@ -98,6 +102,7 @@ require ( github.com/mailru/easyjson v0.9.1 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-pointer v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.17 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect diff --git a/go.sum b/go.sum index 3bec74b1f..8bae31b15 100644 --- a/go.sum +++ b/go.sum @@ -213,6 +213,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= +github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.17 h1:78v8ZlW0bP43XfmAfPsdXcoNCelfMHsDmd/pkENfrjQ= github.com/mattn/go-runewidth v0.0.17/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -315,6 +317,34 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tree-sitter/go-tree-sitter v0.25.0 h1:sx6kcg8raRFCvc9BnXglke6axya12krCJF5xJ2sftRU= +github.com/tree-sitter/go-tree-sitter v0.25.0/go.mod h1:r77ig7BikoZhHrrsjAnv8RqGti5rtSyvDHPzgTPsUuU= +github.com/tree-sitter/tree-sitter-c v0.23.4 h1:nBPH3FV07DzAD7p0GfNvXM+Y7pNIoPenQWBpvM++t4c= +github.com/tree-sitter/tree-sitter-c v0.23.4/go.mod h1:MkI5dOiIpeN94LNjeCp8ljXN/953JCwAby4bClMr6bw= +github.com/tree-sitter/tree-sitter-cpp v0.23.4 h1:LaWZsiqQKvR65yHgKmnaqA+uz6tlDJTJFCyFIeZU/8w= +github.com/tree-sitter/tree-sitter-cpp v0.23.4/go.mod h1:doqNW64BriC7WBCQ1klf0KmJpdEvfxyXtoEybnBo6v8= +github.com/tree-sitter/tree-sitter-embedded-template v0.23.2 h1:nFkkH6Sbe56EXLmZBqHHcamTpmz3TId97I16EnGy4rg= +github.com/tree-sitter/tree-sitter-embedded-template v0.23.2/go.mod h1:HNPOhN0qF3hWluYLdxWs5WbzP/iE4aaRVPMsdxuzIaQ= +github.com/tree-sitter/tree-sitter-go v0.23.4 h1:yt5KMGnTHS+86pJmLIAZMWxukr8W7Ae1STPvQUuNROA= +github.com/tree-sitter/tree-sitter-go v0.23.4/go.mod h1:Jrx8QqYN0v7npv1fJRH1AznddllYiCMUChtVjxPK040= +github.com/tree-sitter/tree-sitter-html v0.23.2 h1:1UYDV+Yd05GGRhVnTcbP58GkKLSHHZwVaN+lBZV11Lc= +github.com/tree-sitter/tree-sitter-html v0.23.2/go.mod h1:gpUv/dG3Xl/eebqgeYeFMt+JLOY9cgFinb/Nw08a9og= +github.com/tree-sitter/tree-sitter-java v0.23.5 h1:J9YeMGMwXYlKSP3K4Us8CitC6hjtMjqpeOf2GGo6tig= +github.com/tree-sitter/tree-sitter-java v0.23.5/go.mod h1:NRKlI8+EznxA7t1Yt3xtraPk1Wzqh3GAIC46wxvc320= +github.com/tree-sitter/tree-sitter-javascript v0.23.1 h1:1fWupaRC0ArlHJ/QJzsfQ3Ibyopw7ZfQK4xXc40Zveo= +github.com/tree-sitter/tree-sitter-javascript v0.23.1/go.mod h1:lmGD1EJdCA+v0S1u2fFgepMg/opzSg/4pgFym2FPGAs= +github.com/tree-sitter/tree-sitter-json v0.24.8 h1:tV5rMkihgtiOe14a9LHfDY5kzTl5GNUYe6carZBn0fQ= +github.com/tree-sitter/tree-sitter-json v0.24.8/go.mod h1:F351KK0KGvCaYbZ5zxwx/gWWvZhIDl0eMtn+1r+gQbo= +github.com/tree-sitter/tree-sitter-php v0.23.11 h1:iHewsLNDmznh8kgGyfWfujsZxIz1YGbSd2ZTEM0ZiP8= +github.com/tree-sitter/tree-sitter-php v0.23.11/go.mod h1:T/kbfi+UcCywQfUNAJnGTN/fMSUjnwPXA8k4yoIks74= +github.com/tree-sitter/tree-sitter-python v0.25.0 h1:O6XD9v8U1LOcRc3cNj9nM7XufrtEBezE6VrpRrHZDf0= +github.com/tree-sitter/tree-sitter-python v0.25.0/go.mod h1:cpdthSy/Yoa28aJFBscFHlGiU+cnSiSh1kuDVtI8YeM= +github.com/tree-sitter/tree-sitter-ruby v0.23.1 h1:T/NKHUA+iVbHM440hFx+lzVOzS4dV6z8Qw8ai+72bYo= +github.com/tree-sitter/tree-sitter-ruby v0.23.1/go.mod h1:kUS4kCCQloFcdX6sdpr8p6r2rogbM6ZjTox5ZOQy8cA= +github.com/tree-sitter/tree-sitter-rust v0.23.2 h1:6AtoooCW5GqNrRpfnvl0iUhxTAZEovEmLKDbyHlfw90= +github.com/tree-sitter/tree-sitter-rust v0.23.2/go.mod h1:hfeGWic9BAfgTrc7Xf6FaOAguCFJRo3RBbs7QJ6D7MI= +github.com/tree-sitter/tree-sitter-typescript v0.23.2 h1:/Odvphn18PniVixb9e97X0DbNVsU6Qocv9mfkyzdXwU= +github.com/tree-sitter/tree-sitter-typescript v0.23.2/go.mod h1:zjzMXT/Ulffel2xfOcAkQQkiAkmgnbtPGlFQw/5X4xA= github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk= github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= diff --git a/internal/sync/adapter/adapter.go b/internal/sync/adapter/adapter.go index d41b749df..492c3e2bb 100644 --- a/internal/sync/adapter/adapter.go +++ b/internal/sync/adapter/adapter.go @@ -36,10 +36,13 @@ type DependencyReporter interface { Dependencies(Document) ([]string, error) } -// Document is one repository file supplied to an adapter. +// Document is one repository file supplied to an adapter. ReadOnly marks a +// file that sync reads but never writes, such as a generated module, so a +// value in it is read-only. type Document struct { - Path string - Data []byte + Path string + Data []byte + ReadOnly bool } // SourceBundle is a primary source and the dependencies that its format diff --git a/internal/sync/adapter/code/code.go b/internal/sync/adapter/code/code.go new file mode 100644 index 000000000..7e91a4834 --- /dev/null +++ b/internal/sync/adapter/code/code.go @@ -0,0 +1,738 @@ +// Package code holds the helpers that the tree-sitter adapters share: parsing +// with a time limit, string codecs, message lists, and placeholder +// translation. No format package imports another format package. +package code + +import ( + "bytes" + "encoding/json" + "fmt" + "maps" + "slices" + "strconv" + "strings" + "time" + "unicode/utf8" + + sitter "github.com/tree-sitter/go-tree-sitter" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +// MaxSourceBytes is the largest source that a code adapter parses. +const MaxSourceBytes = 4 << 20 + +// ParseTree parses a source with a time limit. It rejects a source that is +// not UTF-8, that is too large, or that has syntax errors. +func ParseTree(document adapter.Document, language *sitter.Language) (*sitter.Tree, error) { + if !utf8.Valid(document.Data) { + return nil, fmt.Errorf("%s: source is not valid UTF-8", document.Path) + } + if len(document.Data) > MaxSourceBytes { + return nil, fmt.Errorf("%s: source exceeds %d bytes", document.Path, MaxSourceBytes) + } + parser := sitter.NewParser() + defer parser.Close() + if err := parser.SetLanguage(language); err != nil { + return nil, fmt.Errorf("%s: set parser language: %w", document.Path, err) + } + deadline := time.Now().Add(2 * time.Second) + tree := parser.ParseWithOptions(func(offset int, _ sitter.Point) []byte { + if offset >= len(document.Data) { + return nil + } + return document.Data[offset:] + }, nil, &sitter.ParseOptions{ + ProgressCallback: func(sitter.ParseState) bool { + return time.Now().After(deadline) + }, + }) + if tree == nil { + return nil, fmt.Errorf("%s: parser timed out", document.Path) + } + if tree.RootNode().HasError() { + tree.Close() + return nil, fmt.Errorf("%s: source contains syntax errors", document.Path) + } + return tree, nil +} + +// Walk visits every named node of a tree, parents before children. +func Walk(root *sitter.Node, visit func(*sitter.Node)) { + stack := []*sitter.Node{root} + for len(stack) != 0 { + node := stack[len(stack)-1] + stack = stack[:len(stack)-1] + visit(node) + for index := node.NamedChildCount(); index > 0; index-- { + stack = append(stack, node.NamedChild(index-1)) + } + } +} + +// NamedChildren returns the named children of a node without comments. +func NamedChildren(node *sitter.Node) []*sitter.Node { + if node == nil { + return nil + } + children := make([]*sitter.Node, 0, node.NamedChildCount()) + for index := uint(0); index < node.NamedChildCount(); index++ { + child := node.NamedChild(index) + if child.Kind() != "comment" { + children = append(children, child) + } + } + return children +} + +// FindNode returns the first node of kind whose text is text, or nil. +func FindNode(root *sitter.Node, source []byte, kind, text string) *sitter.Node { + var found *sitter.Node + Walk(root, func(node *sitter.Node) { + if found == nil && node.Kind() == kind && Text(node, source) == text { + found = node + } + }) + return found +} + +// Text returns the source text of a node. +func Text(node *sitter.Node, source []byte) string { + if node == nil { + return "" + } + return string(source[node.StartByte():node.EndByte()]) +} + +// MessageList reads a collection of role and content pairs as a message +// list. A pair can be a two-item list or an object. A message with a dynamic +// role or content makes the list read-only. +func MessageList(collection *adapter.Node) *adapter.Node { + if collection == nil || collection.Items == nil { + return nil + } + result := &adapter.Node{ + Items: collection.Items, Collection: true, Start: collection.Start, End: collection.End, File: collection.File, + Capability: adapter.CapabilityDirect, Reason: "message structure is read-only", + } + // A read-only collection, such as a mutated import, keeps its reason, + // so that no publish writes its pre-mutation messages. + if collection.Capability == adapter.CapabilityReadOnly { + result.Capability, result.Reason = adapter.CapabilityReadOnly, collection.Reason + return result + } + var messages []syncdomain.Message + for _, item := range collection.Items { + fields := item.Fields + if fields == nil && len(item.Items) == 2 { + fields = map[string]*adapter.Node{"role": item.Items[0], "content": item.Items[1]} + item.Fields = fields + } + role, roleOK := StringValue(fields["role"]) + content, contentOK := StringValue(fields["content"]) + // The role of a message is structure, and a constructor such as + // HumanMessage gives it. The content must be a writable value. + if !roleOK || !contentOK || !Writable(fields["content"]) { + result.Capability = adapter.CapabilityReadOnly + result.Reason = "dynamic message content is read-only" + return result + } + messages = append(messages, syncdomain.Message{ + Role: NormalizeRole(role), Content: content, + }) + } + result.Value = messages + return result +} + +// DecodePythonString decodes a Python string literal and returns its value +// and its quote. Formatted, byte, and raw strings are read-only. +func DecodePythonString(raw []byte) (string, string, error) { + text := string(raw) + quoteAt := strings.IndexAny(text, `"'`) + if quoteAt < 0 { + return "", "", fmt.Errorf("unsupported Python string") + } + prefix := strings.ToLower(text[:quoteAt]) + if strings.ContainsAny(prefix, "fbr") || strings.Trim(prefix, "u") != "" { + return "", "", fmt.Errorf("Python string prefix %q is read-only", prefix) + } + quote := text[quoteAt : quoteAt+1] + if strings.HasPrefix(text[quoteAt:], quote+quote+quote) { + quote = quote + quote + quote + } + if len(text) < quoteAt+2*len(quote) || !strings.HasSuffix(text, quote) { + return "", "", fmt.Errorf("invalid Python string") + } + value, err := decodeEscapes(text[quoteAt+len(quote):len(text)-len(quote)], true) + return NormalizeLineEndings(value), quote, err +} + +// NormalizeLineEndings gives a string literal one line ending, so a prompt +// in a file with Windows or classic Mac line endings reads as the same text +// everywhere, as the prompt normalization of a variation does. +func NormalizeLineEndings(value string) string { + return strings.ReplaceAll(strings.ReplaceAll(value, "\r\n", "\n"), "\r", "\n") +} + +// MatchLineEndings writes an encoded literal with the line endings of the +// literal that it replaces, so a file keeps one line ending throughout. +func MatchLineEndings(encoded, previous []byte) []byte { + if !bytes.Contains(previous, []byte("\r\n")) { + return encoded + } + normalized := bytes.ReplaceAll(encoded, []byte("\r\n"), []byte("\n")) + return bytes.ReplaceAll(normalized, []byte("\n"), []byte("\r\n")) +} + +// EncodePythonString writes a value as a Python string with the given quote. +func EncodePythonString(value, quote string) []byte { + body := encodeEscapes(value, quote[0], len(quote) == 3) + if len(quote) == 3 { + body = strings.ReplaceAll(body, quote, `\`+quote) + } + return []byte(quote + body + quote) +} + +// DecodeJSString decodes a JavaScript string literal and returns its value +// and its quote. A template with interpolation is read-only. +func DecodeJSString(raw []byte) (string, byte, error) { + if len(raw) < 2 || raw[0] != raw[len(raw)-1] { + return "", 0, fmt.Errorf("invalid TypeScript string") + } + quote := raw[0] + if quote != '\'' && quote != '"' && quote != '`' { + return "", 0, fmt.Errorf("unsupported TypeScript string") + } + body := string(raw[1 : len(raw)-1]) + if quote == '`' && strings.Contains(body, "${") { + return "", 0, fmt.Errorf("template interpolation is read-only") + } + value, err := decodeEscapes(body, false) + return NormalizeLineEndings(value), quote, err +} + +// EncodeJSString writes a value as a JavaScript string with the given quote. +// A template literal holds its line breaks as they are, so a multi-line +// prompt keeps its layout. +func EncodeJSString(value string, quote byte) ([]byte, error) { + if quote == '`' && strings.Contains(value, "${") { + return nil, fmt.Errorf("template interpolation is read-only") + } + return []byte(string(quote) + encodeEscapes(value, quote, quote == '`') + string(quote)), nil +} + +func decodeEscapes(text string, python bool) (string, error) { + var result strings.Builder + for offset := 0; offset < len(text); { + if text[offset] != '\\' { + r, width := utf8.DecodeRuneInString(text[offset:]) + result.WriteRune(r) + offset += width + continue + } + offset++ + if offset == len(text) { + return "", fmt.Errorf("string ends with a backslash") + } + escape := text[offset] + offset++ + switch escape { + case '\\', '\'', '"', '`': + result.WriteByte(escape) + case 'n': + result.WriteByte('\n') + case 'r': + result.WriteByte('\r') + case 't': + result.WriteByte('\t') + case 'b': + result.WriteByte('\b') + case 'f': + result.WriteByte('\f') + case 'v': + result.WriteByte('\v') + case '\n': + // A backslash before a line break joins the lines. + case '\r': + // A Windows line break is two bytes, and the continuation + // takes both. + if offset < len(text) && text[offset] == '\n' { + offset++ + } + case 'x', 'u', 'U': + digits := map[byte]int{'x': 2, 'u': 4, 'U': 8}[escape] + if !python && escape == 'U' { + return "", fmt.Errorf("TypeScript does not support \\U escapes") + } + if offset+digits > len(text) { + return "", fmt.Errorf("short hexadecimal escape") + } + value, err := strconv.ParseUint(text[offset:offset+digits], 16, 32) + if err != nil || value > utf8.MaxRune || value >= 0xd800 && value <= 0xdfff { + return "", fmt.Errorf("invalid Unicode escape") + } + result.WriteRune(rune(value)) + offset += digits + case 'a': + if !python { + return "", fmt.Errorf("unsupported TypeScript escape \\a") + } + result.WriteByte('\a') + case 'N': + if !python { + return "", fmt.Errorf("unsupported TypeScript escape \\N") + } + // A named character needs the Unicode name table, which the + // adapter does not carry, so the string is read-only. + return "", fmt.Errorf("Python escape \\N{...} is not supported") + case '0', '1', '2', '3', '4', '5', '6', '7': + if !python { + return "", fmt.Errorf("unsupported TypeScript escape \\%c", escape) + } + // An octal escape has one to three digits. + digits := string(escape) + for len(digits) < 3 && offset < len(text) && text[offset] >= '0' && text[offset] <= '7' { + digits += string(text[offset]) + offset++ + } + value, _ := strconv.ParseUint(digits, 8, 32) + if value > 0o377 { + return "", fmt.Errorf("Python octal escape \\%s is out of range", digits) + } + result.WriteRune(rune(value)) + default: + if !python { + return "", fmt.Errorf("unsupported TypeScript escape \\%c", escape) + } + result.WriteByte('\\') + result.WriteByte(escape) + } + } + return result.String(), nil +} + +func encodeEscapes(value string, quote byte, multiline bool) string { + var result strings.Builder + for _, character := range value { + switch character { + case '\\': + result.WriteString(`\\`) + case '\n': + if multiline { + result.WriteByte('\n') + } else { + result.WriteString(`\n`) + } + case '\r': + result.WriteString(`\r`) + case '\t': + result.WriteString(`\t`) + case rune(quote): + result.WriteByte('\\') + result.WriteRune(character) + default: + if character < 0x20 { + fmt.Fprintf(&result, `\u%04x`, character) + } else { + result.WriteRune(character) + } + } + } + return result.String() +} + +// PythonLiteral writes a JSON-compatible value as a Python literal. +// Strings use double quotes. Objects keep their key order sorted, as the +// JSON encoding does. +func PythonLiteral(value any) ([]byte, error) { + switch typed := value.(type) { + case nil: + return []byte("None"), nil + case bool: + if typed { + return []byte("True"), nil + } + return []byte("False"), nil + case string: + return EncodePythonString(typed, `"`), nil + case float64, int, int64: + return json.Marshal(typed) + case []any: + items, err := pythonItems(typed) + if err != nil { + return nil, err + } + return []byte("[" + strings.Join(items, ", ") + "]"), nil + case map[string]any: + pairs := make([]string, 0, len(typed)) + for _, key := range slices.Sorted(maps.Keys(typed)) { + encoded, err := PythonLiteral(typed[key]) + if err != nil { + return nil, err + } + pairs = append(pairs, string(EncodePythonString(key, `"`))+": "+string(encoded)) + } + return []byte("{" + strings.Join(pairs, ", ") + "}"), nil + default: + return nil, fmt.Errorf("value of type %T has no Python literal", value) + } +} + +// UnresolvedImport is the read-only node of an imported value that static +// inspection cannot follow. file names the module that holds it, so that a +// hint can tell the user to link that file. +func UnresolvedImport(module, name, file string) *adapter.Node { + node := adapter.ReadOnly(0, 0, fmt.Sprintf("imported value %s.%s cannot be resolved statically", module, name)) + node.Code = syncdomain.ErrorImportUnresolved + node.File = file + return node +} + +// AmbiguousImport returns the read-only value of a name that two imports of +// one scope bind, as in a try/except fallback, because no one module is its +// value. file is the file of the first import, so that the name still counts +// as an import. +func AmbiguousImport(name, file string) *adapter.Node { + node := adapter.ReadOnly(0, 0, fmt.Sprintf("%s is imported more than once, so its value is not static", name)) + node.Code = syncdomain.ErrorImportUnresolved + node.File = file + return node +} + +// LocateConstants records the file of every constant of a dependency. A +// dependency that sync never writes, such as a generated module, makes each +// of its values read-only, so a mapped field in it blocks a write. +func LocateConstants(constants map[string]*adapter.Node, document adapter.Document) { + for _, value := range constants { + value.SetFile(document.Path) + if document.ReadOnly && value != nil { + value.MarkReadOnly(document.Path + " is generated, vendored, or minified, so sync does not write it") + value.Code = syncdomain.ErrorSourceNotWritable + } + } +} + +// Constant returns a top-level literal as a logical resource with an id, so +// that constants[id="NAME"] can select it. +func Constant(id string, value *adapter.Node) *adapter.Node { + fields := make(map[string]*adapter.Node, len(value.Fields)+1) + for key, child := range value.Fields { + fields[key] = child + } + // The id of the resource is the variable name. An object entry named id + // is not reachable through the resource, and a map of the id says so + // instead of reading the variable name as a value. + fields["id"] = &adapter.Node{ + Value: id, Capability: adapter.CapabilityReadOnly, + Reason: "the id of a constant is its variable name, not a value of the object", + } + return &adapter.Node{ + Value: value.Value, Fields: fields, Items: value.Items, Collection: value.Collection, + Start: value.Start, End: value.End, File: value.File, Capability: value.Capability, + Reason: value.Reason, Code: value.Code, Encode: value.Encode, + } +} + +// Receiver finds the instance that a dotted callable is called on, and the +// method after it. A receiver can be a name or an attribute chain, such as +// self.client or this.client, so the longest prefix that names an instance +// wins. kindOf returns the kind of a receiver, or an empty string. +func Receiver(callable string, kindOf func(string) string) (kind, method string) { + for index := strings.LastIndex(callable, "."); index > 0; index = strings.LastIndex(callable[:index], ".") { + if kind := kindOf(callable[:index]); kind != "" { + return kind, callable[index+1:] + } + } + return "", "" +} + +// SplitMethod splits a callable at its last dot, so that the owner of +// p.ChatPromptTemplate.fromMessages is p.ChatPromptTemplate. A namespace +// import names a class through the namespace, so the owner can have dots. +func SplitMethod(callable string) (owner, method string, qualified bool) { + index := strings.LastIndex(callable, ".") + if index < 0 { + return callable, "", false + } + return callable[:index], callable[index+1:], true +} + +// PythonTuple writes a list of values as a Python tuple, so that a tuple +// constant stays a tuple on a write. A tuple of one item keeps its comma. +func PythonTuple(value any) ([]byte, error) { + typed, ok := value.([]any) + if !ok { + return nil, adapter.TypeMismatch("source", "a list") + } + items, err := pythonItems(typed) + if err != nil { + return nil, err + } + if len(items) == 1 { + return []byte("(" + items[0] + ",)"), nil + } + return []byte("(" + strings.Join(items, ", ") + ")"), nil +} + +func pythonItems(values []any) ([]string, error) { + items := make([]string, len(values)) + for index, item := range values { + encoded, err := PythonLiteral(item) + if err != nil { + return nil, err + } + items[index] = string(encoded) + } + return items, nil +} + +// NormalizeRole maps framework role names to the LaunchDarkly roles. +func NormalizeRole(role string) string { + switch role { + case "human": + return syncdomain.RoleUser + case "ai": + return syncdomain.RoleAssistant + default: + return role + } +} + +// FallbackIDs gives resources that have no name of their own distinct ids. +// The first resource that falls back to an id keeps it, and each later one +// gets an ordinal suffix, so that a keyed selector matches one resource. +type FallbackIDs struct { + seen map[string]int +} + +// NewFallbackIDs returns an empty allocator. +func NewFallbackIDs() *FallbackIDs { return &FallbackIDs{seen: map[string]int{}} } + +// Distinguish marks the id of a resource that fell back to its position as +// positional, and renames it when an earlier resource already used it. +func (ids *FallbackIDs) Distinguish(resource *adapter.Node, fallback string) { + if id, _ := StringValue(resource.Fields["id"]); id != fallback { + return + } + ids.seen[fallback]++ + id := fallback + if count := ids.seen[fallback]; count > 1 { + id = fmt.Sprintf("%s#%d", fallback, count) + } + resource.Fields["id"] = adapter.PositionalID(id) +} + +// DistinguishDuplicates gives each of the resources that share an id an +// ordinal, in order, as a positional id, so that no two resources of one +// collection share one id and a selector names one of them. +func DistinguishDuplicates(resources []*adapter.Node) { + counts := map[string]int{} + for _, resource := range resources { + if id, ok := StringValue(resource.Fields["id"]); ok { + counts[id]++ + } + } + ordinals := map[string]int{} + for _, resource := range resources { + id, ok := StringValue(resource.Fields["id"]) + if !ok || counts[id] < 2 { + continue + } + ordinals[id]++ + resource.Fields["id"] = adapter.PositionalID(fmt.Sprintf("%s#%d", id, ordinals[id])) + } +} + +// SameNode reports whether two handles name the same syntax node. Tree-sitter +// returns a new handle on each access, so the handles themselves differ. +func SameNode(left, right *sitter.Node) bool { + return left != nil && right != nil && left.Kind() == right.Kind() && + left.StartByte() == right.StartByte() && left.EndByte() == right.EndByte() +} + +// StringValue returns the string value of a node. A nil node has none. +func StringValue(node *adapter.Node) (string, bool) { + if node == nil { + return "", false + } + text, ok := node.Value.(string) + return text, ok +} + +// HasValue reports whether a node has a static value. A null leaf has a nil +// Value and is still static, so a collection that holds one stays writable. +func HasValue(node *adapter.Node) bool { + return node != nil && (node.Value != nil || node.Capability == adapter.CapabilityDirect) +} + +// Writable reports whether a node has a static value that a collection can +// hold and write back: a direct leaf, a null leaf included, or a direct +// object or collection. A read-only node, such as a mutated import, makes the +// collection around it read-only. +func Writable(node *adapter.Node) bool { + return node != nil && node.Capability == adapter.CapabilityDirect +} + +// IsLiteral reports whether a node is a literal that a constant can hold: a +// static value, an object, or a collection. +func IsLiteral(node *adapter.Node) bool { + return HasValue(node) || node != nil && (node.Fields != nil || node.Items != nil) +} + +// ShadowKey returns the key under which a constants map keeps an imported +// value that a local declaration shadows. The space keeps it apart from +// every qualified id. +func ShadowKey(name string) string { return "import " + name } + +// RecordShadowedImport keeps an imported value that a module-level +// declaration shadows, with the start of the first such declaration, so that +// a use before it still resolves the import. +func RecordShadowedImport(constants map[string]*adapter.Node, name string, value *adapter.Node, declarationStart int) { + if value == nil { + return + } + if _, recorded := constants[ShadowKey(name)]; recorded { + return + } + constants[ShadowKey(name)] = &adapter.Node{ + Start: declarationStart, Fields: map[string]*adapter.Node{"import": value}, + } +} + +// ShadowedImport returns a copy of the import that a use of a module-level +// name sees before the local declaration that shadows it, or nil. +func ShadowedImport(constants map[string]*adapter.Node, name string, use int) *adapter.Node { + shadow := constants[ShadowKey(name)] + if shadow == nil || use >= shadow.Start { + return nil + } + return shadow.Fields["import"].Clone() +} + +// ImportSiteKey is the key under which the constants of a file record where +// an import statement binds a name. +func ImportSiteKey(name string) string { return "import site " + name } + +// RecordImportSite records the end of the import statement that binds a name +// in the primary file, so that a use above it in the same scope does not see +// the imported value. The first statement of a name wins. +func RecordImportSite(constants map[string]*adapter.Node, name string, end int) { + if _, recorded := constants[ImportSiteKey(name)]; !recorded { + constants[ImportSiteKey(name)] = &adapter.Node{End: end} + } +} + +// ImportBound reports whether an imported name is bound at a byte offset of +// the primary file. A name without a recorded site is bound everywhere. +func ImportBound(constants map[string]*adapter.Node, name string, use int) bool { + site := constants[ImportSiteKey(name)] + return site == nil || site.End <= use +} + +// FrozenKey is the key under which the constants of a file keep the first +// literal of a name that the file reassigns or mutates. +func FrozenKey(id string) string { return "frozen " + id } + +// RecordFrozen keeps the read-only first literal of a reassigned or mutated +// name, so that an import of the name binds the literal as read-only instead +// of reading as unresolved. The first declaration wins. +func RecordFrozen(constants map[string]*adapter.Node, id string, value *adapter.Node) { + if _, recorded := constants[FrozenKey(id)]; !recorded { + constants[FrozenKey(id)] = value + } +} + +// Frozen returns the read-only first literal of a reassigned or mutated +// name, or nil when the file declares no literal for it. +func Frozen(constants map[string]*adapter.Node, id string) *adapter.Node { + return constants[FrozenKey(id)] +} + +// AliasKey is the key under which the constants of a file record that a name +// is an alias of another constant, as `ALSO = ITEMS` makes it. +func AliasKey(id string) string { return "alias " + id } + +// RecordAlias records that id binds the object of origin, so that an import +// of either name binds one value and a mutation through one freezes both. +func RecordAlias(constants map[string]*adapter.Node, id, origin string) { + constants[AliasKey(id)] = &adapter.Node{Value: origin} +} + +// AliasOrigin returns the name whose object id binds, following a chain of +// aliases, or id itself when it is not an alias. +func AliasOrigin(constants map[string]*adapter.Node, id string) string { + for range len(constants) + 1 { + alias := constants[AliasKey(id)] + if alias == nil { + return id + } + origin, _ := alias.Value.(string) + if origin == "" || origin == id { + return id + } + id = origin + } + return id +} + +// FirstDeclarationKey is the key under which the constants of a file record +// where the first declaration of a name starts. +func FirstDeclarationKey(id string) string { return "first " + id } + +// RecordFirstDeclaration records where the first declaration of a name in the +// primary file starts. The first record wins. +func RecordFirstDeclaration(constants map[string]*adapter.Node, id string, start int) { + if _, recorded := constants[FirstDeclarationKey(id)]; !recorded { + constants[FirstDeclarationKey(id)] = &adapter.Node{Start: start} + } +} + +// DeclaredAfter reports whether every declaration of a name in the primary +// file starts after a byte offset, so that a use at the offset comes before +// the scope binds the name at all. +func DeclaredAfter(constants map[string]*adapter.Node, id string, use int) bool { + first := constants[FirstDeclarationKey(id)] + return first != nil && first.Start > use +} + +// DeclarationKey is the key under which the constants of a file record where +// a declaration of the primary file ends. +func DeclarationKey(id string) string { return "declared " + id } + +// RecordDeclaration records where a declaration of the primary file ends, so +// that a value copied from an import is in scope after the declaration only. +func RecordDeclaration(constants map[string]*adapter.Node, id string, end int) { + constants[DeclarationKey(id)] = &adapter.Node{End: end} +} + +// DeclaredAt reports whether a constant is in scope at a byte offset of the +// primary file. A declaration of the primary file must end before the use, a +// value copied from an import included. An import without a declaration here +// is in scope everywhere, and its statement position has its own check. +func DeclaredAt(constants map[string]*adapter.Node, id string, constant *adapter.Node, use int) bool { + if constant == nil { + return false + } + if declaration := constants[DeclarationKey(id)]; declaration != nil { + return declaration.End < use + } + return Declared(constant, use) +} + +// Declared reports whether a constant is in scope at a byte offset of the +// primary file. A constant from another file is in scope everywhere. A +// constant of the primary file must end before the use. +func Declared(constant *adapter.Node, use int) bool { + return constant != nil && (constant.File != "" || constant.End < use) +} + +// StaticString returns the string value of a node, or an empty string. +func StaticString(node *adapter.Node) string { + if node == nil { + return "" + } + value, _ := node.Value.(string) + return value +} diff --git a/internal/sync/adapter/code/code_test.go b/internal/sync/adapter/code/code_test.go new file mode 100644 index 000000000..7714bc0c4 --- /dev/null +++ b/internal/sync/adapter/code/code_test.go @@ -0,0 +1,210 @@ +package code + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +func TestStringCodecsHandleSupportedEscapes(t *testing.T) { + python := []byte(`"\\\'\"\n\r\t\b\f\v\x41\u0042\U0001F600"`) + value, _, err := DecodePythonString(python) + require.NoError(t, err) + // A carriage return escape decodes to the one line ending. + assert.Equal(t, "\\'\"\n\n\t\b\f\vAB😀", value) + + typescript := []byte(`"\\\'\"\n\r\t\b\f\v\x41\u0042"`) + value, _, err = DecodeJSString(typescript) + require.NoError(t, err) + assert.Equal(t, "\\'\"\n\n\t\b\f\vAB", value) + + for name, input := range map[string][]byte{ + "short": {'"'}, + "bad delimiter": []byte(`x`), + "template expression": []byte("`hello ${name}`"), + "trailing slash": []byte(`"hello\"`), + "bad hex": []byte(`"\xGG"`), + "TypeScript U escape": []byte(`"\U00000041"`), + } { + t.Run(name, func(t *testing.T) { + if input[0] == '`' || name == "TypeScript U escape" { + _, _, err := DecodeJSString(input) + require.Error(t, err) + return + } + _, _, err := DecodePythonString(input) + require.Error(t, err) + }) + } +} + +func TestStringCodecsPreserveQuotesControlsAndMultilineText(t *testing.T) { + assert.Equal(t, "\"\"\"line\n\\\"\\\"\\\"end\"\"\"", string(EncodePythonString("line\n\"\"\"end", `"""`))) + encoded, err := EncodeJSString("line\n\t\r\x01\"", '"') + require.NoError(t, err) + assert.Equal(t, `"line\n\t\r\u0001\""`, string(encoded)) + _, err = EncodeJSString("unsafe ${value}", '`') + require.Error(t, err) + + for name, input := range map[string][]byte{ + "raw prefix": []byte(`r"text"`), + "byte prefix": []byte(`b"text"`), + "formatted prefix": []byte(`f"text"`), + "unknown prefix": []byte(`q"text"`), + "unterminated triple": []byte(`"""text"`), + "short hex": []byte(`"\x1"`), + "invalid Unicode": []byte(`"\uD800"`), + "large Unicode": []byte(`"\UFFFFFFFF"`), + } { + t.Run(name, func(t *testing.T) { + _, _, err := DecodePythonString(input) + require.Error(t, err) + }) + } +} + +func TestPlaceholderTranslationRejectsAmbiguousForms(t *testing.T) { + tests := []struct { + name string + value string + style PlaceholderStyle + from bool + }{ + {name: "brace escaped", value: "{{name}}", style: BracePlaceholders}, + {name: "brace residue", value: "literal }", style: BracePlaceholders}, + {name: "source unclosed", value: "{name", style: BracePlaceholders}, + {name: "source bad name", value: "{1name}", style: BracePlaceholders}, + {name: "dollar unclosed", value: "${name", style: DollarPlaceholders}, + {name: "target residue", value: "literal {", style: BracePlaceholders, from: true}, + {name: "target residue before placeholder", value: "literal { {{name}}", style: BracePlaceholders, from: true}, + {name: "target unclosed", value: "{{name", style: BracePlaceholders, from: true}, + {name: "target bad name", value: "{{1name}}", style: BracePlaceholders, from: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var err error + if test.from { + _, err = placeholdersFromLaunchDarkly(test.value, test.style) + } else { + _, err = placeholdersToLaunchDarkly(test.value, test.style) + } + require.Error(t, err) + }) + } + + node := adapter.String("hello {name}", 0, 14, func(text string) ([]byte, error) { return []byte(text), nil }) + translated := TranslatePlaceholders(node, BracePlaceholders) + _, err := translated.Encode(1) + require.Error(t, err) + _, err = translated.Encode("hello {{1name}}") + require.Error(t, err) + assert.Nil(t, TranslatePlaceholders(nil, BracePlaceholders)) + + dynamic := &adapter.Node{Value: true} + assert.Same(t, dynamic, TranslatePlaceholders(dynamic, BracePlaceholders)) + assert.Same(t, dynamic, translateMessagePlaceholders(dynamic, BracePlaceholders)) +} + +func TestJSStringCodecRejectsMalformedInput(t *testing.T) { + _, _, err := DecodeJSString([]byte("xx")) + require.Error(t, err) + _, _, err = DecodeJSString([]byte(`"x'`)) + require.Error(t, err) + _, err = decodeEscapes("\\\nnext", false) + require.NoError(t, err) + _, err = decodeEscapes(`\q`, false) + require.Error(t, err) +} + +func TestLineContinuationsJoinWindowsLineBreaks(t *testing.T) { + assert.Equal(t, "a\nb\nc", NormalizeLineEndings("a\r\nb\rc"), "Windows and classic Mac line endings become one") + value, _, err := DecodePythonString([]byte("\"one \\\r\ntwo\"")) + require.NoError(t, err) + assert.Equal(t, "one two", value, "a backslash before CRLF joins the lines as Python does") + + value, _, err = DecodeJSString([]byte("'one \\\r\ntwo'")) + require.NoError(t, err) + assert.Equal(t, "one two", value) + + value, _, err = DecodePythonString([]byte("\"one \\\ntwo\"")) + require.NoError(t, err) + assert.Equal(t, "one two", value) +} + +func TestMessageListKeepsTheFileOfItsSource(t *testing.T) { + collection := &adapter.Node{ + File: "src/values.py", Start: 4, End: 40, Collection: true, + Items: []*adapter.Node{{Fields: map[string]*adapter.Node{ + "role": {Value: "user"}, "content": {Value: "Hi"}, + }}}, + } + + messages := MessageList(collection) + + require.NotNil(t, messages) + assert.Equal(t, "src/values.py", messages.File, "an imported message list keeps the dependency file") + assert.Equal(t, 4, messages.Start) +} + +func TestConstantIDIsTheVariableName(t *testing.T) { + value := &adapter.Node{ + Value: map[string]any{"id": "abc", "prompt": "Help"}, + Fields: map[string]*adapter.Node{ + "id": {Value: "abc", Capability: adapter.CapabilityDirect}, + "prompt": {Value: "Help", Capability: adapter.CapabilityDirect}, + }, + Capability: adapter.CapabilityDirect, + } + + constant := Constant("CONFIG", value) + + assert.Equal(t, "CONFIG", constant.Fields["id"].Value, "the resource is selected by its variable name") + assert.Equal(t, adapter.CapabilityReadOnly, constant.Fields["id"].Capability, "a map of the id says that it is the variable name") + assert.Contains(t, constant.Fields["id"].Reason, "variable name") + assert.Equal(t, "Help", constant.Fields["prompt"].Value) + assert.Equal(t, "abc", constant.Value.(map[string]any)["id"], "a write of the whole object keeps the entry") +} + +func TestPythonEscapesCoverOctalBellAndNamedCharacters(t *testing.T) { + value, _, err := DecodePythonString([]byte(`"\101\x41\a\7"`)) + require.NoError(t, err) + assert.Equal(t, "AA\a\a", value, "octal escapes and the bell escape decode as Python does") + + _, _, err = DecodePythonString([]byte(`"\N{BULLET}"`)) + require.Error(t, err, "a named character is read-only instead of wrong") +} + +func TestPythonOctalEscapesStopAtThreeSevenSeven(t *testing.T) { + _, _, err := DecodePythonString([]byte(`"\400"`)) + require.Error(t, err, "an octal escape above \\377 is read-only instead of wrong") + value, _, err := DecodePythonString([]byte(`"\377"`)) + require.NoError(t, err) + assert.Equal(t, "\u00ff", value) +} + +func TestLocateConstantsMakesTheValuesOfAReadOnlyDependencyReadOnly(t *testing.T) { + authored := map[string]*adapter.Node{ + "TEXT": adapter.String("Help", 7, 13, nil), + "GONE": nil, + } + LocateConstants(authored, adapter.Document{Path: "app/prompt.py"}) + assert.Equal(t, "app/prompt.py", authored["TEXT"].File) + assert.Equal(t, adapter.CapabilityDirect, authored["TEXT"].Capability, "an authored dependency stays writable") + + generated := map[string]*adapter.Node{ + "TEXT": adapter.String("Help", 7, 13, nil), + "ITEMS": {Items: []*adapter.Node{adapter.String("A", 1, 4, nil)}, Collection: true, Capability: adapter.CapabilityDirect}, + "GONE": nil, + } + LocateConstants(generated, adapter.Document{Path: "app/schema_pb2.py", ReadOnly: true}) + assert.Equal(t, "app/schema_pb2.py", generated["TEXT"].File) + assert.Equal(t, adapter.CapabilityReadOnly, generated["TEXT"].Capability) + assert.Equal(t, syncdomain.ErrorSourceNotWritable, generated["TEXT"].Code) + assert.Equal(t, "app/schema_pb2.py is generated, vendored, or minified, so sync does not write it", generated["TEXT"].Reason) + assert.Equal(t, adapter.CapabilityReadOnly, generated["ITEMS"].Items[0].Capability, "every value below the constant is read-only too") + assert.Nil(t, generated["GONE"], "a reassigned name stays unresolved") +} diff --git a/internal/sync/adapter/code/placeholder.go b/internal/sync/adapter/code/placeholder.go new file mode 100644 index 000000000..a99557d5f --- /dev/null +++ b/internal/sync/adapter/code/placeholder.go @@ -0,0 +1,214 @@ +package code + +import ( + "fmt" + "strings" + "unicode" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" +) + +// PlaceholderStyle is the placeholder syntax of a framework. LaunchDarkly +// uses {{name}}. Translation is exact or the value is read-only. +type PlaceholderStyle string + +const ( + BracePlaceholders PlaceholderStyle = "brace" + DollarPlaceholders PlaceholderStyle = "dollar" +) + +// TranslatePlaceholders returns a copy of a string node whose value uses +// LaunchDarkly placeholders and whose encoder writes them back in style. +func TranslatePlaceholders(node *adapter.Node, style PlaceholderStyle) *adapter.Node { + if node == nil { + return nil + } + value, ok := node.Value.(string) + if !ok { + return node + } + translated, err := placeholdersToLaunchDarkly(value, style) + if err != nil { + result := *node + result.Value = nil + result.Capability = adapter.CapabilityReadOnly + result.Reason = err.Error() + result.Code = syncdomain.ErrorPlaceholderAmbiguous + result.Encode = nil + return &result + } + result := *node + result.Value = translated + encode := node.Encode + result.Encode = func(value any) ([]byte, error) { + text, ok := value.(string) + if !ok { + return nil, adapter.TypeMismatch("source", "a string") + } + sourceText, err := placeholdersFromLaunchDarkly(text, style) + if err != nil { + return nil, err + } + return encode(sourceText) + } + return &result +} + +func translateMessagePlaceholders(node *adapter.Node, style PlaceholderStyle) *adapter.Node { + if node == nil || node.Items == nil { + return node + } + // A read-only list, such as a mutated import, keeps its reason, because + // nothing writes it. + if node.Capability == adapter.CapabilityReadOnly { + return node + } + result := *node + result.Items = make([]*adapter.Node, len(node.Items)) + var messages []syncdomain.Message + for index, item := range node.Items { + copied := *item + copied.Fields = make(map[string]*adapter.Node, len(item.Fields)) + for key, value := range item.Fields { + copied.Fields[key] = value + } + translated := TranslatePlaceholders(item.Fields["content"], style) + if translated != nil { + copied.Fields["content"] = translated + } + result.Items[index] = &copied + role, roleOK := StringValue(copied.Fields["role"]) + content, contentOK := StringValue(translated) + if !roleOK || !contentOK { + result.Value = nil + result.Capability = adapter.CapabilityReadOnly + result.Reason, result.Code = "dynamic message content is read-only", "" + if translated != nil && translated.Reason != "" { + result.Reason, result.Code = translated.Reason, translated.Code + } + return &result + } + messages = append(messages, syncdomain.Message{ + Role: NormalizeRole(role), Content: content, + }) + } + result.Value = messages + return &result +} + +// TranslateFields translates the placeholders of the named fields in place. +// A messages field translates the content of each message. +func TranslateFields( + fields map[string]*adapter.Node, + style PlaceholderStyle, + names ...string, +) map[string]*adapter.Node { + for _, name := range names { + if node := fields[name]; node != nil { + if name == "messages" { + fields[name] = translateMessagePlaceholders(node, style) + } else { + fields[name] = TranslatePlaceholders(node, style) + } + } + } + return fields +} + +func placeholdersToLaunchDarkly(value string, style PlaceholderStyle) (string, error) { + var output strings.Builder + for index := 0; index < len(value); { + start, end, name, ok, err := nextSourcePlaceholder(value, index, style) + if err != nil { + return "", err + } + if !ok { + output.WriteString(value[index:]) + break + } + output.WriteString(value[index:start]) + output.WriteString("{{" + name + "}}") + index = end + } + return output.String(), nil +} + +func nextSourcePlaceholder(value string, offset int, style PlaceholderStyle) (int, int, string, bool, error) { + token := "{" + if style == DollarPlaceholders { + token = "${" + } + index := strings.Index(value[offset:], token) + if index < 0 { + if style == BracePlaceholders && strings.ContainsAny(value[offset:], "{}") { + return 0, 0, "", false, fmt.Errorf("literal or escaped braces make placeholder translation ambiguous") + } + return 0, 0, "", false, nil + } + index += offset + // A brace in the text before the placeholder is a literal or an escape. + if style == BracePlaceholders && strings.ContainsAny(value[offset:index], "{}") { + return 0, 0, "", false, fmt.Errorf("literal or escaped braces make placeholder translation ambiguous") + } + if style == BracePlaceholders && (strings.HasPrefix(value[index:], "{{") || index > 0 && value[index-1] == '{') { + return 0, 0, "", false, fmt.Errorf("escaped braces make placeholder translation ambiguous") + } + nameStart := index + len(token) + closeAt := strings.IndexByte(value[nameStart:], '}') + if closeAt < 0 { + return 0, 0, "", false, fmt.Errorf("placeholder is not closed") + } + closeAt += nameStart + name := value[nameStart:closeAt] + if !validPlaceholderName(name) { + return 0, 0, "", false, fmt.Errorf("placeholder %q is ambiguous", name) + } + return index, closeAt + 1, name, true, nil +} + +func placeholdersFromLaunchDarkly(value string, style PlaceholderStyle) (string, error) { + var output strings.Builder + for index := 0; index < len(value); { + start := strings.Index(value[index:], "{{") + if start < 0 { + if strings.ContainsAny(value[index:], "{}") { + return "", fmt.Errorf("literal braces make placeholder translation ambiguous") + } + output.WriteString(value[index:]) + break + } + start += index + // A brace in the text before the placeholder would become a field. + if strings.ContainsAny(value[index:start], "{}") { + return "", fmt.Errorf("literal braces make placeholder translation ambiguous") + } + output.WriteString(value[index:start]) + closeAt := strings.Index(value[start+2:], "}}") + if closeAt < 0 { + return "", fmt.Errorf("LaunchDarkly placeholder is not closed") + } + closeAt += start + 2 + name := value[start+2 : closeAt] + if !validPlaceholderName(name) { + return "", fmt.Errorf("LaunchDarkly placeholder %q is ambiguous", name) + } + if style == DollarPlaceholders { + output.WriteString("${" + name + "}") + } else { + output.WriteString("{" + name + "}") + } + index = closeAt + 2 + } + return output.String(), nil +} + +func validPlaceholderName(name string) bool { + for index, character := range name { + if character == '_' || unicode.IsLetter(character) || index > 0 && (unicode.IsDigit(character) || character == '.' || character == '-') { + continue + } + return false + } + return name != "" +} diff --git a/internal/sync/adapter/python/frameworks.go b/internal/sync/adapter/python/frameworks.go new file mode 100644 index 000000000..3f36c2cc8 --- /dev/null +++ b/internal/sync/adapter/python/frameworks.go @@ -0,0 +1,301 @@ +package python + +import ( + "strings" + + sitter "github.com/tree-sitter/go-tree-sitter" + + "github.com/launchdarkly/ldcli/internal/sync/adapter" + "github.com/launchdarkly/ldcli/internal/sync/adapter/code" +) + +type pythonImportRule struct { + module string + symbols map[string]string +} + +var pythonFrameworkAllowlist = []pythonImportRule{ + {module: "agents", symbols: map[string]string{"Agent": "openai-agent", "function_tool": "tool-decorator"}}, + {module: "openai", symbols: map[string]string{"OpenAI": "openai-client"}}, + {module: "anthropic", symbols: map[string]string{"Anthropic": "anthropic-client"}}, + {module: "langchain_core.prompts", symbols: map[string]string{ + "PromptTemplate": "prompt-template", "ChatPromptTemplate": "chat-prompt", + }}, + {module: "langchain_core.tools", symbols: map[string]string{ + "tool": "tool-decorator", "StructuredTool": "structured-tool", + }}, + {module: "langgraph.prebuilt", symbols: map[string]string{"create_react_agent": "react-agent"}}, + {module: "crewai", symbols: map[string]string{"Agent": "crewai-agent", "Task": "crewai-task"}}, + {module: "pydantic_ai", symbols: map[string]string{"Agent": "pydantic-agent"}}, + {module: "google.adk.agents", symbols: map[string]string{"Agent": "adk-agent", "LlmAgent": "adk-agent"}}, + {module: "strands", symbols: map[string]string{"Agent": "strands-agent"}}, + {module: "strands.tools", symbols: map[string]string{"tool": "tool-decorator"}}, + {module: "autogen_agentchat.agents", symbols: map[string]string{"AssistantAgent": "autogen-agent"}}, + {module: "llama_index.core", symbols: map[string]string{"PromptTemplate": "prompt-template"}}, + {module: "llama_index.core.tools", symbols: map[string]string{"FunctionTool": "function-tool"}}, + {module: "llama_index.core.agent", symbols: map[string]string{"ReActAgent": "llama-react-agent"}}, + {module: "semantic_kernel.agents", symbols: map[string]string{"ChatCompletionAgent": "semantic-agent"}}, + {module: "semantic_kernel.functions", symbols: map[string]string{"kernel_function": "kernel-tool-decorator"}}, + {module: "mcp.server.fastmcp", symbols: map[string]string{"FastMCP": "fastmcp"}}, + {module: "fastmcp", symbols: map[string]string{"FastMCP": "fastmcp"}}, +} + +// pythonCallRule describes the calls of one framework kind. placeholders is +// the placeholder style that the framework fills at run time, so that sync +// translates it to LaunchDarkly placeholders. A framework that sends text as +// it is has none, so braces and JSON in its prompts stay literal. identity +// names the fields that identify a resource when the call is not assigned. +type pythonCallRule struct { + collection string + positional []string + placeholders code.PlaceholderStyle + identity []string +} + +var pythonCallRules = map[string]pythonCallRule{ + "openai-agent": {collection: "agents", positional: []string{"name", "instructions"}}, + "crewai-agent": {collection: "agents", positional: []string{"role", "goal", "backstory"}, placeholders: code.BracePlaceholders, identity: []string{"role"}}, + "crewai-task": {collection: "prompts", positional: []string{"description", "expected_output"}, placeholders: code.BracePlaceholders}, + "pydantic-agent": {collection: "agents", positional: []string{"model"}}, + "adk-agent": {collection: "agents", positional: []string{"name", "model", "instruction", "description"}, placeholders: code.BracePlaceholders}, + "strands-agent": {collection: "agents", positional: []string{"model", "messages", "tools", "system_prompt"}}, + "autogen-agent": {collection: "agents", positional: []string{"name"}}, + "semantic-agent": {collection: "agents", positional: []string{"name", "instructions"}}, + "react-agent": {collection: "agents", positional: []string{"model", "tools", "prompt"}}, + "llama-react-agent": {collection: "agents", positional: []string{"tools"}}, + "prompt": {collection: "prompts", positional: []string{"template"}}, + "function-tool": {collection: "tools", positional: []string{"function"}}, + "structured-tool": {collection: "tools", positional: []string{"function"}}, + "agent-as-tool": {collection: "tools"}, + "openai-request": {collection: "prompts"}, + "anthropic-request": {collection: "prompts"}, +} + +// pythonAllowedImports returns the symbols of an allowlisted framework +// module. Only the exact module matches, so a repository package that +// shares a prefix with a framework, such as agents.prompts, is followed as a +// repository import. +func pythonAllowedImports(module string) map[string]string { + for _, rule := range pythonFrameworkAllowlist { + if module == rule.module { + return rule.symbols + } + } + return nil +} + +// pythonInstances maps each variable or attribute that holds a framework +// client or agent to its kind, by qualified id. A name that two different +// frameworks assign in one scope is ambiguous and has no kind. +func pythonInstances(root *sitter.Node, source []byte, aliases pythonAliases) map[string]string { + instances := map[string]string{} + ambiguous := map[string]bool{} + code.Walk(root, func(node *sitter.Node) { + var name, value *sitter.Node + switch node.Kind() { + case "assignment", "named_expression": + name, value = pythonAssignmentParts(node) + case "with_item": + // `with OpenAI() as client` binds the name to the context value. + if pattern := node.ChildByFieldName("value"); pattern != nil && pattern.Kind() == "as_pattern" { + name, value = pattern.ChildByFieldName("alias"), pythonUnwrap(pattern.NamedChild(0)) + if name != nil && name.NamedChildCount() == 1 { + name = name.NamedChild(0) + } + } + default: + return + } + if name == nil || value == nil { + return + } + // A reset, such as `self.client = None` in a cleanup, does not + // change what kind of client the name holds. + if value.Kind() == "none" { + return + } + // A binding of another value has no kind. It still has an entry, so + // that a local name or a mock shadows an instance of the same name. + kind := "" + if value.Kind() == "call" { + kind = aliases.at(node, source)[code.Text(value.ChildByFieldName("function"), source)] + } + id, ok := pythonInstanceID(node, name, source) + if !ok { + return + } + if previous, seen := instances[id]; seen && previous != kind { + ambiguous[id] = true + } + instances[id] = kind + }) + for id := range ambiguous { + delete(instances, id) + } + return instances +} + +// pythonInstanceID names the variable or attribute that an assignment binds. +// A variable is scoped like a constant. An attribute, such as self.client, +// belongs to the instance of the enclosing class, so every method of the +// class sees it under the class scope. +func pythonInstanceID(assignment, name *sitter.Node, source []byte) (string, bool) { + switch name.Kind() { + case "identifier": + return pythonQualifiedID(assignment, code.Text(name, source), source), true + case "attribute": + return strings.Join(append(pythonClassScopes(assignment, source), code.Text(name, source)), "."), true + } + return "", false +} + +// pythonClassScopes returns the scopes that enclose a node up to its +// innermost class, outermost first. A node outside every class keeps all of +// its scopes. +func pythonClassScopes(node *sitter.Node, source []byte) []string { + var scopes []string + inClass := false + for parent := node.Parent(); parent != nil; parent = parent.Parent() { + kind := parent.Kind() + if kind != "function_definition" && kind != "class_definition" { + continue + } + inClass = inClass || kind == "class_definition" + if !inClass { + continue + } + if scope := parent.ChildByFieldName("name"); scope != nil { + scopes = append([]string{code.Text(scope, source)}, scopes...) + } + } + if !inClass { + return pythonScopes(node, source) + } + return scopes +} + +// pythonInstanceKind returns the kind of the instance that a receiver names +// at a use: the one of the enclosing scope when it exists, else the +// module-level one. self.client in a method also names a class attribute +// `client = OpenAI()` of the class body, as Python resolves it. +func pythonInstanceKind(instances map[string]string, node *sitter.Node, receiver string, source []byte) string { + declared := func(id string) bool { _, ok := instances[id]; return ok } + // A binding in scope decides, with or without a kind, so a local mock + // or a `self.client = mock()` shadows the instance that it hides. + if id := pythonScopedName(node, receiver, source, declared); declared(id) { + return instances[id] + } + for _, prefix := range []string{"self.", "cls."} { + if attribute, ok := strings.CutPrefix(receiver, prefix); ok { + return instances[strings.Join(append(pythonClassScopes(node, source), attribute), ".")] + } + } + return "" +} + +func pythonFrameworkCall( + call *sitter.Node, + id string, + aliases pythonAliases, + instances map[string]string, + source []byte, + constants map[string]*adapter.Node, +) (collection string, resource *adapter.Node, ok bool) { + if call == nil || call.Kind() != "call" { + return "", nil, false + } + callable := code.Text(call.ChildByFieldName("function"), source) + kinds := aliases.at(call, source) + kind := kinds[callable] + if receiver, rest, qualified := strings.Cut(callable, "."); qualified { + ownerKind, method := code.Receiver(callable, func(receiver string) string { + return pythonInstanceKind(instances, call, receiver, source) + }) + if ownerKind == "" { + ownerKind, method = kinds[receiver], rest + } + if ownerKind == "" { + // A module-qualified class, such as tools.StructuredTool.from_function, + // has its kind on the part before the last dot. + if prefix, last, ok := code.SplitMethod(callable); ok && kinds[prefix] != "" { + ownerKind, method = kinds[prefix], last + } + } + switch ownerKind { + case "openai-client": + if method == "chat.completions.create" || method == "responses.create" { + kind = "openai-request" + } + case "anthropic-client": + if method == "messages.create" { + kind = "anthropic-request" + } + case "openai-agent", "pydantic-agent": + if method == "as_tool" { + kind = "agent-as-tool" + } + case "function-tool": + if method == "from_defaults" { + kind = "function-tool" + } + case "structured-tool": + if method == "from_function" { + kind = "structured-tool" + } + case "llama-react-agent": + if method == "from_tools" { + kind = "llama-react-agent" + } + } + } + if kind == "" || isPythonPromptKind(kind) || strings.Contains(kind, "decorator") || + kind == "openai-client" || kind == "anthropic-client" || kind == "fastmcp" { + return "", nil, false + } + fields := pythonKeywordFields(call, source, constants) + positional := pythonPositionalArguments(call) + rule := pythonCallRules[kind] + for index, name := range rule.positional { + if index >= len(positional) { + break + } + // A positional messages list is read as the keyword one is. + if name == "messages" { + fields[name] = pythonMessages(positional[index], source, constants) + continue + } + fields[name] = pythonValue(positional[index], source, constants) + } + if rule.collection == "" { + return "", nil, false + } + if rule.placeholders != "" { + switch rule.collection { + case "agents": + fields = code.TranslateFields( + fields, rule.placeholders, + "instructions", "instruction", "prompt", "system_prompt", "system_message", "goal", "backstory", + ) + case "prompts": + fields = code.TranslateFields( + fields, rule.placeholders, + "instructions", "prompt", "system", "system_prompt", "description", "expected_output", "messages", + ) + } + } + for _, field := range append([]string{"name", "tool_name"}, rule.identity...) { + if name := code.StaticString(fields[field]); name != "" && id == "" { + id = name + } + } + if id == "" { + id = pythonQualifiedCallID(call, callable, source) + } + return rule.collection, adapter.Resource(id, fields), true +} + +// pythonQualifiedCallID names an unassigned call after its callable and every +// enclosing class and function. +func pythonQualifiedCallID(call *sitter.Node, callable string, source []byte) string { + return pythonQualifiedID(call, callable, source) +} diff --git a/internal/sync/adapter/python/python.go b/internal/sync/adapter/python/python.go new file mode 100644 index 000000000..748f4abe0 --- /dev/null +++ b/internal/sync/adapter/python/python.go @@ -0,0 +1,2217 @@ +// Package python inspects Python sources. It finds agents, prompts, tools, and +// constants through an import allowlist and never runs Python. +package python + +import ( + "cmp" + "fmt" + "maps" + "math" + "path" + "slices" + "strconv" + "strings" + + sitter "github.com/tree-sitter/go-tree-sitter" + python "github.com/tree-sitter/tree-sitter-python/bindings/go" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" + "github.com/launchdarkly/ldcli/internal/sync/adapter/code" +) + +type pythonAdapter struct{} + +func (pythonAdapter) Format() string { return "python" } +func (pythonAdapter) Behavior() string { return "python/1" } + +func (pythonAdapter) Inspect(bundle adapter.SourceBundle) (*adapter.Node, error) { + return parsePythonBundle(bundle) +} + +func (pythonAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "", nil } + +func (pythonAdapter) Dependencies(document adapter.Document) ([]string, error) { + tree, err := code.ParseTree(document, sitter.NewLanguage(python.Language())) + if err != nil { + return nil, err + } + defer tree.Close() + var dependencies []string + for module := range pythonImportedModules(tree.RootNode(), document.Data) { + dependencies = append(dependencies, pythonModulePaths(document.Path, module)...) + } + return dependencies, nil +} + +func init() { adapter.MustRegister(pythonAdapter{}) } + +func parsePythonBundle(bundle adapter.SourceBundle) (*adapter.Node, error) { + return parsePython(bundle.Primary, pythonImportedValues(bundle)) +} + +func parsePython(document adapter.Document, imported map[string]*adapter.Node) (*adapter.Node, error) { + tree, err := code.ParseTree(document, sitter.NewLanguage(python.Language())) + if err != nil { + return nil, err + } + defer tree.Close() + + aliases := pythonImports(tree.RootNode(), document.Data) + instances := pythonInstances(tree.RootNode(), document.Data, aliases) + constants, constantResources := pythonConstants(tree.RootNode(), document.Data, imported) + for name, value := range imported { + if _, shadowed := constants[name]; !shadowed { + constants[name] = value + } + } + collections := map[string][]*adapter.Node{"agents": {}, "prompts": {}, "tools": {}, "constants": {}} + collections["constants"] = constantResources + fallbacks := code.NewFallbackIDs() + // A name that holds a resource more than once in its scope gives each + // assignment an ordinal, so that no two resources share one id. + assigned := pythonAssignmentCounts(tree.RootNode(), document.Data) + ordinals := map[string]int{} + code.Walk(tree.RootNode(), func(node *sitter.Node) { + if node.Kind() != "assignment" && node.Kind() != "named_expression" { + return + } + name, value := pythonAssignmentParts(node) + if !pythonNamedTarget(name) || value == nil { + return + } + id := pythonQualifiedID(node, code.Text(name, document.Data), document.Data) + positional := assigned[id] > 1 + if positional { + ordinals[id]++ + id = fmt.Sprintf("%s#%d", id, ordinals[id]) + } + // The framework call of a chain takes its outermost variable name, + // `agent` in `agent = self.backup = Agent()`. A chain of attributes + // only, such as self.agent, leaves the call to the second walk, which + // names it like an unassigned call. + if name.Kind() == "identifier" && !pythonChainTarget(node, "identifier") { + if collection, resource, ok := pythonFrameworkCall( + value, id, aliases, instances, document.Data, constants, + ); ok { + if positional { + resource.Fields["id"] = adapter.PositionalID(id) + } + collections[collection] = append(collections[collection], resource) + } + } + // The inner link of a chain, `b = Agent()` in `a = b = Agent()`, holds + // the prompts that the outer name already collected. An outer link on + // a subscript, `items[0] = b = Agent()`, names nothing, so the inner + // name collects them. A walrus inside a list or a call is not a link + // of the outer assignment, so it names its own prompts. + if pythonChainTarget(node, "identifier") || pythonChainTarget(node, "attribute") { + return + } + // One prompt under the assignment, alone or piped into a chain, takes + // the variable or attribute name. Several take the name with an + // ordinal, so that each has a key of its own. + calls := pythonPromptCalls(value, aliases, document.Data) + for index, call := range calls { + fields := pythonPromptFields(call, aliases, document.Data, constants) + resource := adapter.Resource(id, fields) + switch { + case len(calls) > 1: + // The ordinal is the position of the prompt in the chain. + resource.Fields["id"] = adapter.PositionalID(fmt.Sprintf("%s#%d", id, index+1)) + case positional: + resource.Fields["id"] = adapter.PositionalID(id) + } + collections["prompts"] = append(collections["prompts"], resource) + } + }) + code.Walk(tree.RootNode(), func(node *sitter.Node) { + if node.Kind() != "call" { + return + } + // The first walk named the call that is the whole value of an + // assignment. A call nested in a list or in another call, such as an + // agent in a crew, is a resource of its own. + if !pythonAssignedValue(node) { + if collection, resource, ok := pythonFrameworkCall( + node, "", aliases, instances, document.Data, constants, + ); ok { + callable := code.Text(node.ChildByFieldName("function"), document.Data) + fallbacks.Distinguish(resource, pythonQualifiedCallID(node, callable, document.Data)) + collections[collection] = append(collections[collection], resource) + } + } + // A prompt that a function returns, or builds in place, has no + // variable name, so the enclosing function names it. + if !pythonAssignedCall(node) && pythonPromptCall(node, aliases, document.Data) { + callable := code.Text(node.ChildByFieldName("function"), document.Data) + fields := pythonPromptFields(node, aliases, document.Data, constants) + id := pythonQualifiedCallID(node, callable, document.Data) + resource := adapter.Resource(id, fields) + fallbacks.Distinguish(resource, id) + collections["prompts"] = append(collections["prompts"], resource) + } + }) + code.Walk(tree.RootNode(), func(node *sitter.Node) { + if node.Kind() != "decorated_definition" { + return + } + collection := pythonDecoratedCollection(node, aliases, instances, document.Data) + if collection == "" { + return + } + var function *sitter.Node + for _, child := range code.NamedChildren(node) { + if child.Kind() == "function_definition" { + function = child + break + } + } + if function == nil { + return + } + name := function.ChildByFieldName("name") + if name == nil { + return + } + // The function name is scoped like every other resource, so two + // decorated methods with one name in two classes keep distinct ids. + id := pythonQualifiedID(function, code.Text(name, document.Data), document.Data) + fields := map[string]*adapter.Node{} + // The docstring is the first statement of the body. A comment before + // it is not a statement. + if statements := code.NamedChildren(function.ChildByFieldName("body")); len(statements) != 0 { + first := statements[0] + if first.Kind() == "expression_statement" && first.NamedChildCount() == 1 { + docstring := first.NamedChild(0) + if docstring.Kind() == "string" { + fields["description"] = pythonValue(docstring, document.Data, constants) + } + } + } + for _, child := range code.NamedChildren(node) { + if child.Kind() != "decorator" || child.NamedChildCount() == 0 { + continue + } + decorator := child.NamedChild(0) + if decorator.Kind() == "call" { + for key, value := range pythonKeywordFields(decorator, document.Data, constants) { + fields[key] = value + } + } + } + if parameters := function.ChildByFieldName("parameters"); parameters != nil && parameters.NamedChildCount() != 0 { + fields["schema"] = adapter.ReadOnly( + int(parameters.StartByte()), int(parameters.EndByte()), + "schema inferred from type hints is read-only", + ) + } + if collection == "prompts" { + body := function.ChildByFieldName("body") + fields["template"] = adapter.ReadOnly( + int(body.StartByte()), int(body.EndByte()), + "prompt built by a function is read-only", + ) + } + // The function name is a stable identity, so the id is not positional. + collections[collection] = append(collections[collection], adapter.Resource(id, fields)) + }) + for _, name := range []string{"agents", "prompts", "tools"} { + code.DistinguishDuplicates(collections[name]) + } + return adapter.Collections(collections), nil +} + +// pythonUnwrap returns the expression inside await and parentheses, which +// change no value: `await client.create(...)` or `(Agent(...))`. +func pythonUnwrap(node *sitter.Node) *sitter.Node { + for node != nil { + switch node.Kind() { + case "await": + node = node.NamedChild(0) + case "assignment": + // `a = b = Agent()` nests the second assignment as the value of + // the first, and both names hold the call. + node = node.ChildByFieldName("right") + case "named_expression": + // `a = (b := Agent())` holds the call in the walrus. + node = node.ChildByFieldName("value") + case "parenthesized_expression": + inner := code.NamedChildren(node) + if len(inner) != 1 { + return node + } + node = inner[0] + default: + return node + } + } + return nil +} + +// pythonAssignedValue reports whether a call is the whole value of an +// assignment, with or without await and parentheses around it, which the +// assignment walk names after its variable. +func pythonAssignedValue(call *sitter.Node) bool { + for parent := call.Parent(); parent != nil; parent = parent.Parent() { + switch parent.Kind() { + case "await", "parenthesized_expression": + continue + case "assignment", "named_expression": + left, right := pythonAssignmentParts(parent) + if !code.SameNode(right, call) { + return false + } + // The assignment walk names a call when a variable of the chain + // holds it. A chain of attributes only, such as self.agent, is + // named like an unassigned call. + if left != nil && left.Kind() == "identifier" { + return true + } + default: + return false + } + } + return false +} + +// pythonAssignedCall reports whether a call sits under an assignment that +// names it: one with a variable or an attribute target. A call under a +// subscript or an unpacking target, such as `prompts["greet"] = ...` or +// `first, second = ...`, has no name, so the enclosing function names it. +func pythonAssignedCall(call *sitter.Node) bool { + return pythonNamedAssignment(call.Parent()) != nil +} + +// pythonNamedAssignment returns the nearest assignment that encloses node in +// its statement and has a variable or an attribute target, or nil. +func pythonNamedAssignment(node *sitter.Node) *sitter.Node { + for ; node != nil; node = node.Parent() { + switch node.Kind() { + case "assignment", "named_expression": + if name, _ := pythonAssignmentParts(node); pythonNamedTarget(name) { + return node + } + case "expression_statement", "return_statement": + return nil + } + } + return nil +} + +// pythonDeclaredVariable returns the variable and the value of an assignment +// or a walrus, `name = value` or `name := value`, or nil for another node or +// another target. A walrus in a comprehension binds in the enclosing scope, +// as the assignment does. +func pythonDeclaredVariable(node *sitter.Node) (name, value *sitter.Node) { + switch node.Kind() { + case "assignment": + name, value = node.ChildByFieldName("left"), node.ChildByFieldName("right") + case "named_expression": + name, value = node.ChildByFieldName("name"), node.ChildByFieldName("value") + default: + return nil, nil + } + if name == nil || name.Kind() != "identifier" { + return nil, nil + } + return name, value +} + +// pythonAssignmentCounts counts the assignments to each variable or +// attribute by qualified id, so that a name assigned more than once in its +// scope is known before the resources are collected. +func pythonAssignmentCounts(root *sitter.Node, source []byte) map[string]int { + counts := map[string]int{} + code.Walk(root, func(node *sitter.Node) { + if node.Kind() != "assignment" && node.Kind() != "named_expression" { + return + } + if name, _ := pythonAssignmentParts(node); pythonNamedTarget(name) { + counts[pythonQualifiedID(node, code.Text(name, source), source)]++ + } + }) + return counts +} + +// pythonAssignmentParts returns the target and the unwrapped value of an +// assignment, `name = value`, or of a walrus, `name := value`. Another node +// has neither. +func pythonAssignmentParts(node *sitter.Node) (name, value *sitter.Node) { + switch node.Kind() { + case "assignment": + return node.ChildByFieldName("left"), pythonUnwrap(node.ChildByFieldName("right")) + case "named_expression": + return node.ChildByFieldName("name"), pythonUnwrap(node.ChildByFieldName("value")) + } + return nil, nil +} + +// pythonChainTarget reports whether an outer link of the assignment chain +// of node has a target of the kind. A chain is `a = b = Agent()`, where each +// outer assignment holds the inner one as its whole value. A walrus inside a +// list or a call, as in `x = [a := Agent()]`, is not a link of a chain. +func pythonChainTarget(node *sitter.Node, kind string) bool { + child := node + for parent := node.Parent(); parent != nil; child, parent = parent, parent.Parent() { + switch parent.Kind() { + case "await", "parenthesized_expression": + continue + case "assignment", "named_expression": + left, right := pythonAssignmentParts(parent) + if !code.SameNode(right, pythonUnwrap(child)) { + return false + } + if left != nil && left.Kind() == kind { + return true + } + default: + return false + } + } + return false +} + +// pythonNamedTarget reports whether an assignment target is a variable or +// an attribute, which names the value that it holds. +func pythonNamedTarget(name *sitter.Node) bool { + return name != nil && (name.Kind() == "identifier" || name.Kind() == "attribute") +} + +// pythonPromptCalls returns the prompt calls that an assigned value holds. +// A call that a walrus inside the value binds belongs to that name, which +// names it on its own. +func pythonPromptCalls(node *sitter.Node, aliases pythonAliases, source []byte) []*sitter.Node { + var calls []*sitter.Node + code.Walk(node, func(candidate *sitter.Node) { + if pythonPromptCall(candidate, aliases, source) && !pythonInnerBinding(candidate, node) { + calls = append(calls, candidate) + } + }) + return calls +} + +// pythonInnerBinding reports whether an assignment or a walrus between a +// call and the value that holds it names the call itself. +func pythonInnerBinding(call, value *sitter.Node) bool { + if code.SameNode(call, value) { + return false + } + for parent := call.Parent(); parent != nil && !code.SameNode(parent, value); parent = parent.Parent() { + if parent.Kind() == "assignment" || parent.Kind() == "named_expression" { + if name, _ := pythonAssignmentParts(parent); pythonNamedTarget(name) { + return true + } + } + } + return false +} + +// pythonPromptCall reports whether a node builds a prompt template of an +// allowlisted framework. +func pythonPromptCall(node *sitter.Node, aliases pythonAliases, source []byte) bool { + if node.Kind() != "call" { + return false + } + callable := code.Text(node.ChildByFieldName("function"), source) + owner, method, qualified := code.SplitMethod(callable) + kinds := aliases.at(node, source) + return isPythonPromptKind(kinds[callable]) || + (qualified && isPythonPromptKind(kinds[owner]) && (method == "from_messages" || method == "from_template")) +} + +func pythonPromptFields( + call *sitter.Node, + aliases pythonAliases, + source []byte, + constants map[string]*adapter.Node, +) map[string]*adapter.Node { + callable := code.Text(call.ChildByFieldName("function"), source) + owner, method, qualified := code.SplitMethod(callable) + arguments := pythonPositionalArguments(call) + kinds := aliases.at(call, source) + if qualified && isPythonPromptKind(kinds[owner]) { + // The factory takes its one argument by position or by keyword. + argument := func(name string) *sitter.Node { + if len(arguments) != 0 { + return arguments[0] + } + return pythonKeywordArgument(call, name, source) + } + switch { + case method == "from_messages" && argument("messages") != nil: + return code.TranslateFields( + map[string]*adapter.Node{"messages": pythonMessages(argument("messages"), source, constants)}, + code.BracePlaceholders, "messages", + ) + case method == "from_template" && argument("template") != nil: + return code.TranslateFields( + map[string]*adapter.Node{"template": pythonValue(argument("template"), source, constants)}, + code.BracePlaceholders, "template", + ) + case len(arguments) == 0: + return map[string]*adapter.Node{} + } + } + fields := pythonKeywordFields(call, source, constants) + if len(arguments) != 0 { + if kinds[callable] == "prompt-template" { + fields["template"] = pythonValue(arguments[0], source, constants) + } else if _, exists := fields["messages"]; !exists { + fields["messages"] = pythonMessages(arguments[0], source, constants) + } + } + return code.TranslateFields(fields, code.BracePlaceholders, "template", "messages") +} + +func isPythonPromptKind(kind string) bool { + return kind == "prompt-template" || kind == "chat-prompt" +} + +// pythonKeywordArgument returns the value of a keyword argument, or nil. +func pythonKeywordArgument(call *sitter.Node, name string, source []byte) *sitter.Node { + for _, argument := range code.NamedChildren(call.ChildByFieldName("arguments")) { + if argument.Kind() == "keyword_argument" && code.Text(argument.ChildByFieldName("name"), source) == name { + return argument.ChildByFieldName("value") + } + } + return nil +} + +// pythonPositionalArguments returns the arguments that a call passes by +// position. A splat, such as *args or **options, fills no positional field. +func pythonPositionalArguments(call *sitter.Node) []*sitter.Node { + var result []*sitter.Node + for _, argument := range code.NamedChildren(call.ChildByFieldName("arguments")) { + switch argument.Kind() { + case "keyword_argument", "list_splat", "dictionary_splat": + continue + } + result = append(result, argument) + } + return result +} + +// pythonAliases are the framework symbols that the imports of a file bind, +// such as Agent to "openai-agent", with the position and the scope of each +// import. Two frameworks can export one name, so a use sees the binding +// that Python sees at that point. +type pythonAliases struct { + bindings []pythonAliasBinding +} + +// pythonAliasBinding is one framework symbol that an import binds. +type pythonAliasBinding struct { + alias, kind string + // scopes are the functions and classes that enclose the import. + scopes []string + end int + // class reports that the innermost scope is a class body, which binds + // the name on the class only. + class bool + // fallback is one more than the start of the try statement whose body + // or except clause holds the import, or zero without one. Two imports + // of one name in one try statement bind either one, as in an import + // with a fallback. + fallback int +} + +// pythonImports returns the framework symbols that the imports of a file +// bind. +func pythonImports(root *sitter.Node, source []byte) pythonAliases { + var bindings []pythonAliasBinding + add := func(node *sitter.Node, alias, kind string) { + chain := pythonScopeChain(node, source) + bindings = append(bindings, pythonAliasBinding{ + alias: alias, kind: kind, scopes: pythonScopes(node, source), end: int(node.EndByte()), + class: len(chain) != 0 && chain[len(chain)-1].class, + fallback: pythonTryStatement(node), + }) + } + for _, site := range pythonImportSites(root, source) { + allowed := pythonAllowedImports(site.binding.module) + if allowed == nil { + continue + } + if site.binding.name == "*" { + // `from agents import *` binds every symbol of the framework. + for _, name := range slices.Sorted(maps.Keys(allowed)) { + add(site.node, name, allowed[name]) + } + continue + } + if kind, ok := allowed[site.binding.name]; ok { + add(site.node, site.binding.alias, kind) + } + } + // `import agents as oa` makes every symbol reachable as oa.Agent. + for _, site := range pythonModuleAliasSites(root, source) { + allowed := pythonAllowedImports(site.module) + for _, name := range slices.Sorted(maps.Keys(allowed)) { + add(site.node, site.alias+"."+name, allowed[name]) + } + } + slices.SortStableFunc(bindings, func(left, right pythonAliasBinding) int { return cmp.Compare(left.end, right.end) }) + return pythonAliases{bindings: bindings} +} + +// at returns the framework symbols that a use at node sees: every +// module-level import when the use is inside a function or a class, the +// module-level imports before the use when it is module-level code, and the +// imports of the enclosing functions before the use. A binding of a nearer +// scope wins over one of an outer scope, and a later binding wins over an +// earlier one of the same scope. +func (aliases pythonAliases) at(node *sitter.Node, source []byte) map[string]string { + use := int(node.StartByte()) + chain := pythonScopeChain(node, source) + scopes := make([]string, len(chain)) + // A function body runs later, so it sees every module-level import. A + // class body runs where it is, like module-level code. + inFunction := pythonInLambda(node) + for index, scope := range chain { + scopes[index] = scope.name + inFunction = inFunction || !scope.class + } + kinds := map[string]string{} + depths := map[string]int{} + fallbacks := map[string]int{} + ambiguous := map[string]bool{} + for _, binding := range aliases.bindings { + if len(binding.scopes) == 0 { + if !inFunction && binding.end > use { + continue + } + } else if len(scopes) < len(binding.scopes) || !slices.Equal(scopes[:len(binding.scopes)], binding.scopes) || binding.end > use { + continue + } else if binding.class && (len(scopes) != len(binding.scopes) || pythonInExpressionScope(node)) { + // A class body binds a name on the class only. A method, a + // lambda, or a comprehension inside the class resolves a bare + // name through the enclosing functions and the module. + continue + } + if depth, seen := depths[binding.alias]; seen && depth > len(binding.scopes) { + continue + } + // Two imports of one name in one try statement, as in an import + // with a fallback, bind either framework, so the name is ambiguous. + if previous, seen := fallbacks[binding.alias]; seen && previous != 0 && previous == binding.fallback && kinds[binding.alias] != binding.kind { + ambiguous[binding.alias] = true + } + kinds[binding.alias], depths[binding.alias], fallbacks[binding.alias] = binding.kind, len(binding.scopes), binding.fallback + } + for alias := range ambiguous { + delete(kinds, alias) + } + return kinds +} + +// pythonTryStatement returns one more than the start of the try statement +// whose body or except clause holds a node within its scope, or zero without +// one. The offset keeps a try statement at the start of the file apart from +// no try statement. +func pythonTryStatement(node *sitter.Node) int { + for parent := node.Parent(); parent != nil; parent = parent.Parent() { + switch parent.Kind() { + case "function_definition", "class_definition": + return 0 + case "try_statement": + return int(parent.StartByte()) + 1 + } + } + return 0 +} + +// pythonImportBinding is one name that a from-import creates. +type pythonImportBinding struct { + module string + name string + alias string +} + +// pythonImportBindings returns the bindings of the modules that are not +// allowlisted frameworks, so that imported constants can resolve. +func pythonImportBindings(root *sitter.Node, source []byte) []pythonImportBinding { + var bindings []pythonImportBinding + for _, binding := range pythonImportClauses(root, source) { + if pythonAllowedImports(binding.module) == nil { + bindings = append(bindings, binding) + } + } + return bindings +} + +// pythonImportClauses returns every binding of every from-import. It reads +// the syntax tree, so a comment or a line break inside the import does not +// change the names. +func pythonImportClauses(root *sitter.Node, source []byte) []pythonImportBinding { + sites := pythonImportSites(root, source) + bindings := make([]pythonImportBinding, len(sites)) + for index, site := range sites { + bindings[index] = site.binding + } + return bindings +} + +// pythonImportSite is one binding of a from-import with its statement. +type pythonImportSite struct { + binding pythonImportBinding + node *sitter.Node +} + +// pythonImportSites returns every binding of every from-import with the +// import statement that creates it. +func pythonImportSites(root *sitter.Node, source []byte) []pythonImportSite { + var bindings []pythonImportSite + code.Walk(root, func(node *sitter.Node) { + if node.Kind() != "import_from_statement" { + return + } + module := code.Text(node.ChildByFieldName("module_name"), source) + if module == "" { + return + } + for index := uint(0); index < node.ChildCount(); index++ { + child := node.Child(index) + if node.FieldNameForChild(uint32(index)) != "name" { + continue + } + binding := pythonImportBinding{module: module} + switch child.Kind() { + case "dotted_name": + binding.name = code.Text(child, source) + binding.alias = binding.name + case "aliased_import": + binding.name = code.Text(child.ChildByFieldName("name"), source) + binding.alias = code.Text(child.ChildByFieldName("alias"), source) + default: + continue + } + bindings = append(bindings, pythonImportSite{binding: binding, node: node}) + } + // `from .values import *` binds every name of the module. The names + // are not known statically, so the binding is the wildcard itself. + for _, child := range code.NamedChildren(node) { + if child.Kind() == "wildcard_import" { + bindings = append(bindings, pythonImportSite{ + binding: pythonImportBinding{module: module, name: "*", alias: "*"}, node: node, + }) + } + } + }) + return bindings +} + +// pythonModuleAliases maps each name that an import statement binds to its +// module, such as "oa" for `import agents as oa`. +// pythonModuleAliasSite is one module that an import statement binds, as +// `import agents` or `import agents as oa`. +type pythonModuleAliasSite struct { + alias, module string + node *sitter.Node +} + +func pythonModuleAliasSites(root *sitter.Node, source []byte) []pythonModuleAliasSite { + var sites []pythonModuleAliasSite + code.Walk(root, func(node *sitter.Node) { + if node.Kind() != "import_statement" { + return + } + for index := uint(0); index < node.ChildCount(); index++ { + child := node.Child(index) + if node.FieldNameForChild(uint32(index)) != "name" { + continue + } + switch child.Kind() { + case "dotted_name": + module := code.Text(child, source) + sites = append(sites, pythonModuleAliasSite{alias: module, module: module, node: node}) + case "aliased_import": + sites = append(sites, pythonModuleAliasSite{ + alias: code.Text(child.ChildByFieldName("alias"), source), + module: code.Text(child.ChildByFieldName("name"), source), + node: node, + }) + } + } + }) + return sites +} + +func pythonImportedModules(root *sitter.Node, source []byte) map[string]struct{} { + modules := map[string]struct{}{} + for _, binding := range pythonImportBindings(root, source) { + modules[binding.module] = struct{}{} + } + return modules +} + +func pythonModulePaths(primary, module string) []string { + directory := path.Dir(primary) + dots := len(module) - len(strings.TrimLeft(module, ".")) + module = strings.TrimLeft(module, ".") + if dots == 0 { + directory = "." + } else { + for level := 1; level < dots; level++ { + directory = path.Dir(directory) + } + } + modulePath := path.Join(directory, strings.ReplaceAll(module, ".", "/")) + return []string{modulePath + ".py", path.Join(modulePath, "__init__.py")} +} + +func pythonImportedValues(bundle adapter.SourceBundle) map[string]*adapter.Node { + result := map[string]*adapter.Node{} + tree, err := code.ParseTree(bundle.Primary, sitter.NewLanguage(python.Language())) + if err != nil { + return result + } + defer tree.Close() + cache := map[string]map[string]*adapter.Node{} + for _, site := range pythonImportSites(tree.RootNode(), bundle.Primary.Data) { + binding := site.binding + if pythonAllowedImports(binding.module) != nil { + continue + } + // An import inside a function binds its name in that function, so + // the value is keyed like a constant of that scope, and from the + // import statement on. + key := pythonQualifiedID(site.node, binding.alias, bundle.Primary.Data) + code.RecordImportSite(result, key, int(site.node.EndByte())) + if previous, duplicate := result[key]; duplicate { + // Two imports bind one name, as in a try/except fallback, so no + // one module is its value. + result[key] = code.AmbiguousImport(binding.alias, previous.File) + continue + } + candidates := pythonModulePaths(bundle.Primary.Path, binding.module) + var document adapter.Document + for _, candidate := range candidates { + if value, ok := bundle.Dependencies[candidate]; ok { + document = value + break + } + } + if document.Path == "" { + result[key] = code.UnresolvedImport(binding.module, binding.name, candidates[0]) + continue + } + constants := cache[document.Path] + if constants == nil { + dependencyTree, parseErr := code.ParseTree(document, sitter.NewLanguage(python.Language())) + if parseErr != nil { + result[key] = code.UnresolvedImport(binding.module, binding.name, document.Path) + continue + } + constants, _ = pythonConstants(dependencyTree.RootNode(), document.Data, nil) + dependencyTree.Close() + code.LocateConstants(constants, document) + cache[document.Path] = constants + } + // An alias in the module, `ALSO = ITEMS`, binds the object of its + // origin, so both names import one value. + name := code.AliasOrigin(constants, binding.name) + value := constants[name] + if value == nil { + // A name that the module reassigns or mutates binds its first + // literal as read-only, so the import resolves but blocks a + // write. + value = code.Frozen(constants, name) + } + if value == nil { + value = code.UnresolvedImport(binding.module, binding.name, document.Path) + } + result[key] = value + } + return result +} + +// pythonDeclaration is one binding of a name in the file: an assignment +// with its value, or a parameter or target without one. +type pythonDeclaration struct { + node *sitter.Node + id string + value *sitter.Node + // moved is true when a global or nonlocal statement binds the name in + // another scope than the one that holds the assignment. The assignment + // runs when its function runs, so the constant has no one value. + moved bool +} + +// pythonConstants returns the literal constants of a file by qualified id, +// and the constant resources that constants[id="NAME"] selects. imported +// holds the values of the static imports. A literal can use an imported +// value or a constant of the same file that is declared before it. +func pythonConstants(root *sitter.Node, source []byte, imported map[string]*adapter.Node) (map[string]*adapter.Node, []*adapter.Node) { + type declaration = pythonDeclaration + var declarations []declaration + counts := map[string]int{} + code.Walk(root, func(node *sitter.Node) { + name, valueNode := pythonDeclaredVariable(node) + if name == nil || valueNode == nil { + return + } + id := pythonBindingID(node, code.Text(name, source), source) + declarations = append(declarations, declaration{ + node: node, id: id, value: valueNode, moved: id != pythonQualifiedID(node, code.Text(name, source), source), + }) + counts[id]++ + }) + // A parameter hides a constant of the same name in its function, so it + // has an entry without a value. + code.Walk(root, func(node *sitter.Node) { + if node.Kind() != "parameters" { + return + } + for _, parameter := range code.NamedChildren(node) { + if name := pythonParameterName(parameter, source); name != "" { + id := pythonQualifiedID(parameter, name, source) + if counts[id] == 0 { + declarations = append(declarations, declaration{node: parameter, id: id, value: nil}) + counts[id]++ + } + } + } + }) + // A bare annotation, `TEXT: str`, in a function binds its name as a + // local without a value, so it hides an outer constant as a parameter + // does. An assignment of the name is the binding when there is one. In + // a module or a class body the annotation binds nothing. + code.Walk(root, func(node *sitter.Node) { + if node.Kind() != "assignment" || node.ChildByFieldName("right") != nil || !pythonFunctionLocal(node, source) { + return + } + if left := node.ChildByFieldName("left"); left != nil && left.Kind() == "identifier" { + id := pythonBindingID(node, code.Text(left, source), source) + if counts[id] == 0 { + declarations = append(declarations, declaration{node: node, id: id, value: nil}) + counts[id]++ + } + } + }) + // A loop, with, except, or unpacking target binds its name in the + // enclosing scope, so it hides an outer constant from there on, and a + // constant that it rebinds is read-only. A def or a class binds its name + // the same way. In a function, `NAME += value` and `del NAME` bind the + // name as a local of the whole function too. + code.Walk(root, func(node *sitter.Node) { + for _, target := range pythonRebindingTargets(node) { + name := code.Text(target, source) + if !pythonLocalRebinding(node, name, source) { + continue + } + id := pythonBindingID(node, name, source) + declarations = append(declarations, declaration{ + node: target, id: id, value: nil, moved: id != pythonQualifiedID(node, name, source), + }) + counts[id]++ + } + if name := pythonDefinedName(node); name != nil { + id := pythonBindingID(node, code.Text(name, source), source) + declarations = append(declarations, declaration{node: node, id: id, value: nil}) + counts[id]++ + return + } + for _, target := range pythonBindingTargets(node) { + for _, name := range pythonBoundNames(target, source) { + id := pythonBindingID(target, name, source) + declarations = append(declarations, declaration{node: target, id: id, value: nil}) + counts[id]++ + } + } + }) + // The declarations are read in source order, so a parameter hides a name + // before the body of its function reads it. + slices.SortStableFunc(declarations, func(left, right declaration) int { + return cmp.Compare(left.node.StartByte(), right.node.StartByte()) + }) + // An import binds a name in its scope as a declaration does, so a + // mutation of it is recorded under the same scoped id. + declared := func(id string) bool { return counts[id] != 0 || imported[id] != nil } + mutated := map[string]bool{} + code.Walk(root, func(node *sitter.Node) { + switch node.Kind() { + case "assignment", "augmented_assignment", "named_expression": + // An assignment or a walrus to a variable is a declaration, + // which counts records. Every other write mutates a receiver + // chain. + if declared, _ := pythonDeclaredVariable(node); declared != nil { + return + } + name := node.ChildByFieldName("left") + if name == nil { + return + } + for _, target := range pythonWriteTargets(name) { + // A variable of an unpacking is bound, not mutated, and + // `NAME += value` in a function rebinds a local. + if target.Kind() == "identifier" && + (node.Kind() == "assignment" || pythonLocalRebinding(node, code.Text(target, source), source)) { + continue + } + if id := pythonMutatedName(node, code.Text(target, source), source, declared); id != "" { + mutated[id] = true + } + } + case "call": + function := node.ChildByFieldName("function") + if function == nil { + return + } + owner, method, ok := code.SplitMethod(code.Text(function, source)) + if !ok || !pythonMutatingMethods[method] { + return + } + if id := pythonMutatedName(node, owner, source, declared); id != "" { + mutated[id] = true + } + case "delete_statement": + // `del ITEMS[0]` and `del CONFIG.key` mutate the collection, and + // `del NAME` unbinds the name, so the constant has no one value. + for _, target := range code.NamedChildren(node) { + for _, item := range pythonDeleteTargets(target) { + // `del NAME` in a function unbinds a local. + if item.Kind() == "identifier" && pythonLocalRebinding(node, code.Text(item, source), source) { + continue + } + if id := pythonMutatedName(node, code.Text(item, source), source, declared); id != "" { + mutated[id] = true + } + } + } + } + }) + // An alias shares its object with its origin: `ALSO = ITEMS` binds the + // same list, and two imports of one value bind the same object. A + // mutation through any name of the object freezes every name. + pythonSpreadMutations(mutated, declarations, counts, imported, source, declared) + + // Each literal is read in source order with the imports and every earlier + // constant that is declared once and not mutated in scope, so a chain of + // aliases resolves. A declaration that is not a literal keeps a nil + // entry, so that it still shadows an import or a module-level constant + // of the same name. + environment := maps.Clone(imported) + if environment == nil { + environment = map[string]*adapter.Node{} + } + values := map[string]*adapter.Node{} + // An import that this file mutates is read-only here, as a local + // constant is, and the read-only copy shadows the import. + for name, value := range imported { + if mutated[name] && value != nil { + frozen := value.Clone() + frozen.MarkReadOnly("mutated constants are read-only") + environment[name], values[name] = frozen, frozen + } + } + // A name that a function binds anywhere is local in the whole function, + // so a use of it before the binding is unbound, not the outer constant. + // An entry without a value hides the outer name until the binding gives + // it one. A class body and the module run in order, so they keep the + // outer name until the binding. + for _, item := range declarations { + if _, bound := environment[item.id]; !bound && pythonFunctionLocal(item.node, source) { + environment[item.id] = nil + } + } + var resources []*adapter.Node + fallbacks := code.NewFallbackIDs() + for _, item := range declarations { + code.RecordFirstDeclaration(values, item.id, int(item.node.StartByte())) + // A declaration of an imported name shadows the import from here on, + // and the import stays visible to the uses before it: a module-level + // or a function-local import of the same scope, or the module-level + // import that a class body sees until it binds the name itself. A + // loop, with, except, or unpacking target shadows it the same way. + shadowed := imported[item.id] + if shadowed == nil && pythonInClassBody(item.node) { + if bare := item.id[strings.LastIndex(item.id, ".")+1:]; bare != item.id { + shadowed = imported[bare] + } + } + code.RecordShadowedImport(values, item.id, shadowed, int(item.node.StartByte())) + if item.value == nil { + // A parameter or a target has no value and hides an outer name. + values[item.id], environment[item.id] = nil, nil + continue + } + if item.value.Kind() == "identifier" && counts[item.id] == 1 && !item.moved { + // `ALSO = ITEMS` binds the object of ITEMS, so an import of + // either name binds one value. The record comes before the + // literal check, because a mutated origin has no literal here. + origin := pythonScopedName(item.value, code.Text(item.value, source), source, declared) + if origin != item.id && declared(origin) { + code.RecordAlias(values, item.id, origin) + } + } + value := pythonValue(item.value, source, environment) + if !code.IsLiteral(value) { + // The nil entries shadow an import of the same name here and in + // every later literal. + values[item.id], environment[item.id] = nil, nil + continue + } + if counts[item.id] == 1 && !mutated[item.id] && !item.moved { + environment[item.id] = value + } + values[item.id] = value + code.RecordDeclaration(values, item.id, int(item.node.EndByte())) + resource := code.Constant(item.id, value) + resources = append(resources, resource) + // A name that is assigned more than once has no one value, so each + // of its assignments is a read-only constant with an id of its own. + // A global or nonlocal assignment rebinds the name from another + // function, so the name has no one value either. + if counts[item.id] != 1 || item.moved { + values[item.id] = nil + resource.MarkReadOnly("reassigned constants are read-only") + } + if mutated[item.id] { + values[item.id] = nil + resource.MarkReadOnly("mutated constants are read-only") + } + if counts[item.id] != 1 || mutated[item.id] || item.moved { + // An import of the name binds its first literal as read-only. + frozen := value.Clone() + frozen.MarkReadOnly(resource.Reason) + code.RecordFrozen(values, item.id, frozen) + } + // The positional id comes last, so that no read-only mark replaces it. + if counts[item.id] != 1 { + fallbacks.Distinguish(resource, item.id) + } + } + return values, resources +} + +// pythonDeleteTargets returns each target of a delete statement: the target +// itself, or each item of a tuple or list of targets. +func pythonDeleteTargets(target *sitter.Node) []*sitter.Node { + switch target.Kind() { + case "expression_list", "tuple", "list", "parenthesized_expression": + var targets []*sitter.Node + for _, child := range code.NamedChildren(target) { + targets = append(targets, pythonDeleteTargets(child)...) + } + return targets + } + return []*sitter.Node{target} +} + +// pythonSpreadMutations marks every name that shares an object with a +// mutated name: a declaration that binds a bare name to another constant, +// and the imports that bind one value of another file. +func pythonSpreadMutations( + mutated map[string]bool, + declarations []pythonDeclaration, + counts map[string]int, + imported map[string]*adapter.Node, + source []byte, + declared func(string) bool, +) { + shared := map[string][]string{} + link := func(left, right string) { + shared[left] = append(shared[left], right) + shared[right] = append(shared[right], left) + } + for _, item := range declarations { + // An alias shares its object only while it is bound once: a name + // that is assigned again holds another object later. + if item.value == nil || item.value.Kind() != "identifier" || counts[item.id] != 1 || item.moved { + continue + } + origin := pythonScopedName(item.value, code.Text(item.value, source), source, declared) + if origin != item.id && declared(origin) { + link(item.id, origin) + } + } + byValue := map[*adapter.Node][]string{} + for name, value := range imported { + if value != nil && value.Code == "" { + byValue[value] = append(byValue[value], name) + } + } + for _, names := range byValue { + for _, name := range names[1:] { + link(names[0], name) + } + } + queue := slices.Collect(maps.Keys(mutated)) + for len(queue) != 0 { + name := queue[len(queue)-1] + queue = queue[:len(queue)-1] + for _, other := range shared[name] { + if !mutated[other] { + mutated[other] = true + queue = append(queue, other) + } + } + } +} + +// pythonLocalBinding reports whether a name is a parameter of a lambda +// around it, or a target of a comprehension around it. Both bind the name +// for the expression only, so the name is a value of the call there. A +// default of the lambda and the first iterable of the comprehension run in +// the enclosing scope, so a use there sees the outer name. +func pythonLocalBinding(node *sitter.Node, name string, source []byte) bool { + for parent := node.Parent(); parent != nil; parent = parent.Parent() { + switch parent.Kind() { + case "function_definition", "class_definition": + return false + case "lambda": + if pythonInParameterDefault(node, parent) { + continue + } + for _, parameter := range code.NamedChildren(parent.ChildByFieldName("parameters")) { + if pythonParameterName(parameter, source) == name { + return true + } + } + case "list_comprehension", "set_comprehension", "dictionary_comprehension", "generator_expression": + if pythonInFirstIterable(node, parent) { + continue + } + for _, clause := range code.NamedChildren(parent) { + if clause.Kind() == "for_in_clause" && slices.Contains(pythonBoundNames(clause.ChildByFieldName("left"), source), name) { + return true + } + } + } + } + return false +} + +// pythonInFirstIterable reports whether a node sits in the iterable of the +// first for clause of a comprehension, which Python evaluates in the +// enclosing scope before the comprehension binds its targets. +func pythonInFirstIterable(node, comprehension *sitter.Node) bool { + for _, clause := range code.NamedChildren(comprehension) { + if clause.Kind() != "for_in_clause" { + continue + } + right := clause.ChildByFieldName("right") + return right != nil && node.StartByte() >= right.StartByte() && node.EndByte() <= right.EndByte() + } + return false +} + +// pythonFunctionLocal reports whether the innermost scope of a node is a +// function, where a binding is local in the whole function. +func pythonFunctionLocal(node *sitter.Node, source []byte) bool { + chain := pythonScopeChain(node, source) + return len(chain) != 0 && !chain[len(chain)-1].class +} + +// pythonInExpressionScope reports whether a lambda or a comprehension +// encloses a node within its function or class, or within the module. +func pythonInExpressionScope(node *sitter.Node) bool { + for parent := node.Parent(); parent != nil; parent = parent.Parent() { + switch parent.Kind() { + case "function_definition", "class_definition": + return false + case "lambda", "list_comprehension", "set_comprehension", "dictionary_comprehension", "generator_expression": + return true + } + } + return false +} + +// pythonInLambda reports whether a lambda encloses a node within its +// function or class, or within the module. The body of a lambda runs when +// the lambda is called, after the code around it. +func pythonInLambda(node *sitter.Node) bool { + for parent := node.Parent(); parent != nil; parent = parent.Parent() { + switch parent.Kind() { + case "function_definition", "class_definition": + return false + case "lambda": + // A default of the lambda runs with the definition. + if !pythonInParameterDefault(node, parent) { + return true + } + } + } + return false +} + +// pythonBindingTargets returns the patterns that a statement binds in its +// scope besides an assignment to one variable: the target of a for loop or +// of an unpacking assignment, the alias of a with item or an except clause, +// and the captures of a case pattern. +func pythonBindingTargets(node *sitter.Node) []*sitter.Node { + switch node.Kind() { + case "for_statement": + if left := node.ChildByFieldName("left"); left != nil { + return []*sitter.Node{left} + } + case "assignment": + if left := node.ChildByFieldName("left"); left != nil && left.Kind() != "identifier" { + return []*sitter.Node{left} + } + case "with_item", "except_clause", "except_group_clause": + // The grammar parses `except*` as an except clause too, and a + // grammar with a group clause of its own binds the alias the same way. + if value := node.ChildByFieldName("value"); value != nil && value.Kind() == "as_pattern" { + if alias := value.ChildByFieldName("alias"); alias != nil { + return []*sitter.Node{alias} + } + } + case "case_clause": + // A case pattern captures names for the clause and the scope. + var patterns []*sitter.Node + for _, child := range code.NamedChildren(node) { + if child.Kind() == "case_pattern" { + patterns = append(patterns, child) + } + } + return patterns + } + return nil +} + +// pythonWriteTargets returns the targets that an assignment writes: the +// target itself, or each target of a tuple or list pattern. +func pythonWriteTargets(pattern *sitter.Node) []*sitter.Node { + switch pattern.Kind() { + case "pattern_list", "tuple_pattern", "list_pattern", "parenthesized_expression": + var targets []*sitter.Node + for _, child := range code.NamedChildren(pattern) { + targets = append(targets, pythonWriteTargets(child)...) + } + return targets + } + return []*sitter.Node{pattern} +} + +// pythonBoundNames returns the names that a target pattern binds: the +// identifier, or every identifier of a tuple or list pattern. An attribute +// or a subscript target binds no name. +func pythonBoundNames(pattern *sitter.Node, source []byte) []string { + if pattern == nil { + return nil + } + children := code.NamedChildren(pattern) + switch pattern.Kind() { + case "identifier": + return []string{code.Text(pattern, source)} + case "pattern_list", "tuple_pattern", "list_pattern", "tuple", "list", "as_pattern_target", + "list_splat_pattern", "parenthesized_expression", + "case_pattern", "as_pattern", "splat_pattern", "union_pattern": + var names []string + for _, child := range children { + names = append(names, pythonBoundNames(child, source)...) + } + return names + case "dotted_name": + // A bare name in a case pattern captures the subject. A dotted + // name, or the wildcard, matches a value and binds nothing. + if len(children) == 1 && children[0].Kind() == "identifier" && code.Text(children[0], source) != "_" { + return []string{code.Text(children[0], source)} + } + return nil + case "class_pattern", "keyword_pattern": + // The first child names the class or the keyword, not a capture. + var names []string + for _, child := range children[min(1, len(children)):] { + names = append(names, pythonBoundNames(child, source)...) + } + return names + case "dict_pattern": + var names []string + for _, child := range children { + if child.Kind() == "case_pattern" || child.Kind() == "splat_pattern" { + names = append(names, pythonBoundNames(child, source)...) + } + } + return names + } + return nil +} + +// pythonParameterName returns the name that a function parameter binds, or +// an empty string for self and other forms without a name. +func pythonParameterName(parameter *sitter.Node, source []byte) string { + switch parameter.Kind() { + case "identifier": + return code.Text(parameter, source) + case "default_parameter", "typed_default_parameter": + if name := parameter.ChildByFieldName("name"); name != nil && name.Kind() == "identifier" { + return code.Text(name, source) + } + case "typed_parameter", "list_splat_pattern", "dictionary_splat_pattern": + name := parameter.NamedChild(0) + switch { + case name == nil: + return "" + case name.Kind() == "identifier": + return code.Text(name, source) + case name.Kind() == "list_splat_pattern" || name.Kind() == "dictionary_splat_pattern": + // `*args: T` and `**kwargs: T` wrap the splat in the type. + return pythonParameterName(name, source) + } + } + return "" +} + +// pythonMutatingMethods are the list, dictionary, and set methods that +// change their receiver in place. +var pythonMutatingMethods = map[string]bool{ + "append": true, "extend": true, "insert": true, "remove": true, "pop": true, "clear": true, + "sort": true, "reverse": true, "update": true, "setdefault": true, "popitem": true, + "add": true, "discard": true, +} + +// pythonChainRoot splits a receiver chain at its first attribute or +// subscript, so that SETTINGS["values"].update gives SETTINGS and +// self.items[0] gives self and items[0]. +func pythonChainRoot(chain string) (root, rest string) { + index := strings.IndexAny(chain, ".[") + if index < 0 { + return chain, "" + } + rest = chain[index:] + if rest[0] == '.' { + rest = rest[1:] + } + return chain[:index], rest +} + +// pythonMutatedName returns the id of the constant that a write through a +// receiver chain mutates: the root of items.append, SETTINGS["values"].update, +// or items[0] = x, and the class attribute of self.items.append or +// self.items[0] = x. The chain is the receiver of a call or the target of an +// assignment. An empty chain names nothing. +func pythonMutatedName(node *sitter.Node, chain string, source []byte, declared func(string) bool) string { + rootName, rest := pythonChainRoot(strings.TrimSpace(chain)) + if rootName == "" { + return "" + } + if attribute, _ := pythonChainRoot(rest); attribute != "" { + if rootName == "self" || rootName == "cls" { + return strings.Join(append(pythonClassScopes(node, source), attribute), ".") + } + // A write through the class name, as in Bot.items.append(...), + // mutates the class attribute when the file declares it. + if candidate := pythonScopedName(node, rootName+"."+attribute, source, declared); declared(candidate) { + return candidate + } + } + return pythonScopedName(node, rootName, source, declared) +} + +// pythonConstant returns a copy of the constant that a use of name at a byte +// offset refers to, or nil when none is in scope. The copy keeps a later +// change to one use, such as a read-only mark, away from the other uses. +func pythonConstant(node *sitter.Node, name string, use int, source []byte, constants map[string]*adapter.Node) *adapter.Node { + if constants == nil { + return nil + } + declared := func(id string) bool { _, ok := constants[id]; return ok } + return pythonConstantWith(node, name, use, source, constants, declared) +} + +// pythonConstantWith resolves a use among the ids that declared accepts. +func pythonConstantWith( + node *sitter.Node, + name string, + use int, + source []byte, + constants map[string]*adapter.Node, + declared func(string) bool, +) *adapter.Node { + scoped := pythonScopedName(node, name, source, declared) + if scoped != name && pythonInClassBody(node) && code.DeclaredAfter(constants, scoped, use) { + // A class body runs in order, so a use before the class binds the + // name at all sees the enclosing name, as the module constant, + // whatever the class binds later. + except := func(id string) bool { return id != scoped && declared(id) } + if outer := pythonScopedName(node, name, source, except); outer != scoped { + return pythonConstantWith(node, name, use, source, constants, except) + } + } + value := constants[scoped] + if value == nil { + return code.ShadowedImport(constants, scoped, use) + } + deferred := pythonDeferredUse(node, scoped, source) + // An import binds its name from its statement on, so a use above it in + // the same scope is unbound. A function body runs later and sees it. + if value.File != "" && !deferred && !code.ImportBound(constants, scoped, use) { + return nil + } + if code.DeclaredAt(constants, scoped, value, use) || deferred { + return value.Clone() + } + return code.ShadowedImport(constants, scoped, use) +} + +// pythonDeferredUse reports whether a use runs after the scope of the +// constant finished: a function body or a lambda between the scope of the +// constant and the use runs later, so it sees a constant that the file +// declares after it. +func pythonDeferredUse(node *sitter.Node, scoped string, source []byte) bool { + // The scopes between the scope of the constant and the use. + remaining := len(pythonScopeChain(node, source)) - strings.Count(scoped, ".") + for parent := node.Parent(); parent != nil; parent = parent.Parent() { + switch parent.Kind() { + case "lambda": + if pythonInParameterDefault(node, parent) { + // A default of a lambda runs with the definition too. + continue + } + return true + case "function_definition", "class_definition": + if parent.Kind() == "function_definition" && pythonInParameterDefault(node, parent) { + // A default runs with the definition, not with the body. + continue + } + if remaining <= 0 { + return false + } + if parent.Kind() == "function_definition" { + return true + } + remaining-- + } + } + return false +} + +// pythonInParameterDefault reports whether a node sits in the default value +// of a parameter of the function or the lambda. +func pythonInParameterDefault(node, function *sitter.Node) bool { + for _, parameter := range code.NamedChildren(function.ChildByFieldName("parameters")) { + if parameter.Kind() != "default_parameter" && parameter.Kind() != "typed_default_parameter" { + continue + } + value := parameter.ChildByFieldName("value") + if value != nil && node.StartByte() >= value.StartByte() && node.EndByte() <= value.EndByte() { + return true + } + } + return false +} + +// pythonInClassBody reports whether a node sits directly in a class body, +// where code runs in order like module-level code. +func pythonInClassBody(node *sitter.Node) bool { + for parent := node.Parent(); parent != nil; parent = parent.Parent() { + switch parent.Kind() { + case "class_definition": + return true + case "function_definition": + return false + } + } + return false +} + +// pythonScopedName returns the id that a use of name refers to: the constant +// of the nearest enclosing function or class that declares it, else the +// module-level name. declared reports whether an id has a constant. A bare +// name inside a method does not see the attributes of the class, because +// Python resolves it through the enclosing functions and then the module. +// An attribute chain, such as self.client, belongs to the class. +func pythonScopedName(node *sitter.Node, name string, source []byte, declared func(string) bool) string { + bare := !strings.Contains(name, ".") + // A global or nonlocal statement of the innermost function decides the + // scope of a bare name for its reads and mutations as for its writes. + if bare { + if id, ok := pythonDeclaredScopeID(node, name, source); ok { + return id + } + } + scopes := pythonScopeChain(node, source) + // A lambda or a comprehension is a scope of its own, so the class body + // around it is not the innermost scope of a bare name inside it. + nested := bare && pythonInExpressionScope(node) + for end := len(scopes); end > 0; end-- { + if bare && (end < len(scopes) || nested) && scopes[end-1].class { + continue + } + names := make([]string, 0, end+1) + for _, scope := range scopes[:end] { + names = append(names, scope.name) + } + if candidate := strings.Join(append(names, name), "."); declared(candidate) { + return candidate + } + } + return name +} + +// pythonBindingID returns the qualified id of the scope that a binding of +// name at node writes. A `global` statement in the function moves the +// binding to the module, and a `nonlocal` statement moves it to the nearest +// enclosing function. +func pythonBindingID(node *sitter.Node, name string, source []byte) string { + scopes := pythonScopeChain(node, source) + depth := len(scopes) + nonlocal := false + for parent := node.Parent(); parent != nil && depth != 0; parent = parent.Parent() { + if parent.Kind() != "function_definition" && parent.Kind() != "class_definition" { + continue + } + if parent.Kind() == "function_definition" && pythonInParameterDefault(node, parent) { + continue + } + depth-- + if parent.Kind() == "class_definition" { + if nonlocal { + // A nonlocal name skips the class bodies between the + // functions. + continue + } + } else { + switch pythonScopeStatement(parent, name, source) { + case "global_statement": + if nonlocal { + // A global statement of a function between binds the + // name for that function only, which has no local of + // the name, so the nonlocal search continues. + continue + } + return name + case "nonlocal_statement": + nonlocal = true + continue + } + if nonlocal && !pythonFunctionBinds(parent, name, source) { + // A nonlocal name binds in the nearest enclosing function + // that has the name, so a function without it is skipped. + continue + } + } + names := make([]string, 0, depth+2) + for _, scope := range scopes[:depth+1] { + names = append(names, scope.name) + } + return strings.Join(append(names, name), ".") + } + return name +} + +// pythonFunctionBinds reports whether a function binds name itself: as a +// parameter, or as the target of an assignment, a walrus, a loop, a with, an +// except, or an unpacking in its body. A binding in a nested function, class, +// or lambda belongs to that scope. +func pythonFunctionBinds(function *sitter.Node, name string, source []byte) bool { + return pythonFunctionBindings(function, name, source, true) +} + +// pythonFunctionBindsElsewhere reports whether a function binds name through +// something other than an augmented assignment or a delete. +func pythonFunctionBindsElsewhere(function *sitter.Node, name string, source []byte) bool { + return pythonFunctionBindings(function, name, source, false) +} + +func pythonFunctionBindings(function *sitter.Node, name string, source []byte, rebinds bool) bool { + for _, parameter := range code.NamedChildren(function.ChildByFieldName("parameters")) { + if pythonParameterName(parameter, source) == name { + return true + } + } + bound := false + var visit func(node *sitter.Node) + visit = func(node *sitter.Node) { + if bound { + return + } + switch node.Kind() { + case "function_definition", "class_definition": + // A nested def or class binds its name in this function. + if defined := pythonDefinedName(node); defined != nil && code.Text(defined, source) == name { + bound = true + } + return + case "lambda": + return + case "delete_statement": + // `del NAME` binds the name in the function too. + for _, target := range pythonRebindingTargets(node) { + if rebinds && code.Text(target, source) == name { + bound = true + return + } + } + case "assignment", "named_expression": + if left, _ := pythonAssignmentParts(node); left != nil && left.Kind() == "identifier" && code.Text(left, source) == name { + bound = true + return + } + case "augmented_assignment": + // `NAME += value` binds the name in the function too. + if left := node.ChildByFieldName("left"); rebinds && left != nil && left.Kind() == "identifier" && code.Text(left, source) == name { + bound = true + return + } + case "import_statement", "import_from_statement": + // An import binds its alias in the function. + if slices.Contains(pythonImportedNames(node, source), name) { + bound = true + } + return + } + for _, target := range pythonBindingTargets(node) { + if slices.Contains(pythonBoundNames(target, source), name) { + bound = true + return + } + } + for _, child := range code.NamedChildren(node) { + visit(child) + } + } + visit(function.ChildByFieldName("body")) + return bound +} + +// pythonImportedNames returns the names that an import statement binds: +// the alias of each clause, the first segment of `import a.b`, and the name +// of `from m import name`. A star import binds no name of its own. +func pythonImportedNames(statement *sitter.Node, source []byte) []string { + var names []string + for index := uint(0); index < statement.ChildCount(); index++ { + child := statement.Child(index) + if statement.FieldNameForChild(uint32(index)) != "name" { + continue + } + switch child.Kind() { + case "aliased_import": + if alias := child.ChildByFieldName("alias"); alias != nil { + names = append(names, code.Text(alias, source)) + } + case "dotted_name": + text := code.Text(child, source) + if statement.Kind() == "import_statement" { + text, _, _ = strings.Cut(text, ".") + } + names = append(names, text) + } + } + return names +} + +// pythonRebindingTargets returns the bare names that `NAME += value` and +// `del NAME` write, which bind the name in a function. +func pythonRebindingTargets(node *sitter.Node) []*sitter.Node { + var targets []*sitter.Node + switch node.Kind() { + case "augmented_assignment": + if left := node.ChildByFieldName("left"); left != nil && left.Kind() == "identifier" { + targets = append(targets, left) + } + case "delete_statement": + for _, child := range code.NamedChildren(node) { + for _, item := range pythonDeleteTargets(child) { + if item.Kind() == "identifier" { + targets = append(targets, item) + } + } + } + } + return targets +} + +// pythonLocalRebinding reports whether an augmented assignment or a delete +// of a bare name binds a local of a function: Python makes the name local to +// the whole function unless a global statement names it. At module level and +// in a class body the write mutates the constant of that scope. +func pythonLocalRebinding(node *sitter.Node, name string, source []byte) bool { + if !pythonFunctionLocal(node, source) { + return false + } + if id, declared := pythonDeclaredScopeID(node, name, source); declared && id == name { + return false + } + // A function that binds the name in another way, through an import, an + // assignment, or a parameter, has a local that the write mutates. + for parent := node.Parent(); parent != nil; parent = parent.Parent() { + if parent.Kind() == "function_definition" { + return !pythonFunctionBindsElsewhere(parent, name, source) + } + } + return true +} + +// pythonDefinedName returns the name that a def or a class statement binds, +// or nil for another node. +func pythonDefinedName(node *sitter.Node) *sitter.Node { + if node.Kind() != "function_definition" && node.Kind() != "class_definition" { + return nil + } + return node.ChildByFieldName("name") +} + +// pythonScopeStatement returns the kind of the global or nonlocal statement +// of a function that names name, or an empty string. A statement in a +// nested function, class, or lambda belongs to that scope. +func pythonScopeStatement(function *sitter.Node, name string, source []byte) string { + kind := "" + var visit func(node *sitter.Node) + visit = func(node *sitter.Node) { + if kind != "" { + return + } + switch node.Kind() { + case "function_definition", "class_definition", "lambda": + return + case "global_statement", "nonlocal_statement": + for _, child := range code.NamedChildren(node) { + if child.Kind() == "identifier" && code.Text(child, source) == name { + kind = node.Kind() + return + } + } + return + } + for _, child := range code.NamedChildren(node) { + visit(child) + } + } + visit(function.ChildByFieldName("body")) + return kind +} + +// pythonDeclaredScopeID returns the id that a global or nonlocal statement +// of the innermost enclosing function gives a name, and false when the +// function has no such statement for it. +func pythonDeclaredScopeID(node *sitter.Node, name string, source []byte) (string, bool) { + for parent := node.Parent(); parent != nil; parent = parent.Parent() { + switch parent.Kind() { + case "class_definition": + return "", false + case "function_definition": + if pythonInParameterDefault(node, parent) { + continue + } + if pythonScopeStatement(parent, name, source) == "" { + return "", false + } + return pythonBindingID(node, name, source), true + } + } + return "", false +} + +// pythonScope is one class or function that encloses a node. +type pythonScope struct { + name string + class bool +} + +// pythonScopeChain returns the classes and functions that enclose a node, +// outermost first. +func pythonScopeChain(node *sitter.Node, source []byte) []pythonScope { + var scopes []pythonScope + for parent := node.Parent(); parent != nil; parent = parent.Parent() { + if parent.Kind() != "function_definition" && parent.Kind() != "class_definition" { + continue + } + // A parameter default runs when the function is defined, in the + // enclosing scope, so the function is not one of its scopes. + if parent.Kind() == "function_definition" && pythonInParameterDefault(node, parent) { + continue + } + if scope := parent.ChildByFieldName("name"); scope != nil { + scopes = append([]pythonScope{{ + name: code.Text(scope, source), class: parent.Kind() == "class_definition", + }}, scopes...) + } + } + return scopes +} + +// pythonQualifiedID prefixes a name with every enclosing class and function, +// outermost first, so that two methods with one name in two classes keep +// their locals apart. +func pythonQualifiedID(node *sitter.Node, name string, source []byte) string { + return strings.Join(append(pythonScopes(node, source), name), ".") +} + +// pythonScopes returns the names of the classes and functions that enclose +// a node, outermost first. +func pythonScopes(node *sitter.Node, source []byte) []string { + chain := pythonScopeChain(node, source) + scopes := make([]string, len(chain)) + for index, scope := range chain { + scopes[index] = scope.name + } + return scopes +} + +func pythonKeywordFields(call *sitter.Node, source []byte, constants map[string]*adapter.Node) map[string]*adapter.Node { + fields := map[string]*adapter.Node{} + arguments := call.ChildByFieldName("arguments") + for _, argument := range code.NamedChildren(arguments) { + if argument.Kind() != "keyword_argument" { + continue + } + name := argument.ChildByFieldName("name") + value := argument.ChildByFieldName("value") + if name == nil || value == nil { + continue + } + key := code.Text(name, source) + if value.Kind() == "call" && key == "model_settings" && + strings.HasSuffix(code.Text(value.ChildByFieldName("function"), source), "ModelSettings") { + fields[key] = &adapter.Node{Fields: pythonKeywordFields(value, source, constants)} + continue + } + if key == "messages" { + fields[key] = pythonMessages(value, source, constants) + continue + } + fields[key] = pythonValue(value, source, constants) + } + return fields +} + +func pythonValue(node *sitter.Node, source []byte, constants map[string]*adapter.Node) *adapter.Node { + if node == nil { + return adapter.ReadOnly(0, 0, "value is missing") + } + start, end := int(node.StartByte()), int(node.EndByte()) + switch node.Kind() { + case "parenthesized_expression": + if inner := code.NamedChildren(node); len(inner) == 1 { + return pythonValue(inner[0], source, constants) + } + case "assignment": + // `a = b = "text"` nests the second assignment as the value of the first. + if right := node.ChildByFieldName("right"); right != nil { + return pythonValue(right, source, constants) + } + case "named_expression": + // `NAME = (OTHER := "text")` and `[x := "hello"]` hold the value + // of the walrus, as the resource collection reads it. + if value := node.ChildByFieldName("value"); value != nil { + return pythonValue(value, source, constants) + } + case "string": + value, quote, err := code.DecodePythonString(source[start:end]) + if err != nil { + return adapter.ReadOnly(start, end, err.Error()) + } + return adapter.String(value, start, end, func(text string) ([]byte, error) { + return code.MatchLineEndings(code.EncodePythonString(text, quote), source[start:end]), nil + }) + case "concatenated_string": + // Adjacent literals are one string. A write replaces them all with + // one literal in the quote of the first part. + var value, quote string + for _, part := range code.NamedChildren(node) { + text, partQuote, err := code.DecodePythonString([]byte(code.Text(part, source))) + if part.Kind() != "string" || err != nil { + return adapter.ReadOnly(start, end, "string concatenation is not static") + } + if quote == "" { + quote = partQuote + } + value += text + } + return adapter.String(value, start, end, func(text string) ([]byte, error) { + return code.MatchLineEndings(code.EncodePythonString(text, quote), source[start:end]), nil + }) + case "integer", "float", "unary_operator": + value, reason := pythonNumber(node, source) + if reason != "" { + return adapter.ReadOnly(start, end, reason) + } + return adapter.Number(value, start, end, func(number float64) ([]byte, error) { + return []byte(strconv.FormatFloat(number, 'g', -1, 64)), nil + }) + case "true", "false": + return adapter.Bool(node.Kind() == "true", start, end, encodePythonBool) + case "identifier": + switch string(source[start:end]) { + case "True", "False": + return adapter.Bool(string(source[start:end]) == "True", start, end, encodePythonBool) + } + if constants != nil { + // A lambda parameter or a comprehension target is a value of + // the call, not a constant. + if pythonLocalBinding(node, string(source[start:end]), source) { + return adapter.ReadOnly(start, end, "dynamic expression has no safe inverse") + } + if value := pythonConstant(node, string(source[start:end]), start, source, constants); value != nil { + return value + } + declared := func(id string) bool { _, ok := constants[id]; return ok } + // A parameter, a target, or a binding without a literal has an + // entry without a value, so the name is a dynamic local, not a + // name that a star import supplies. + if scoped := pythonScopedName(node, string(source[start:end]), source, declared); declared(scoped) && constants[scoped] == nil { + return adapter.ReadOnly(start, end, "dynamic expression has no safe inverse") + } + if wildcard := constants[pythonScopedName(node, "*", source, declared)]; wildcard != nil { + value := *wildcard + value.Reason = fmt.Sprintf("star import cannot resolve %s statically", code.Text(node, source)) + return &value + } + } + case "none": + return &adapter.Node{Start: start, End: end, Capability: adapter.CapabilityDirect, Encode: code.PythonLiteral} + case "dictionary": + return pythonDictionary(node, source, constants) + case "list", "tuple": + return pythonSequence(node, source, constants) + } + return adapter.ReadOnly(start, end, "dynamic expression has no safe inverse") +} + +// pythonNumber reads an integer or float literal, with a sign, digit +// separators, and a base prefix, as a float64. A value that it cannot read +// comes with the read-only reason: a sign on something other than a literal +// is a dynamic expression, and an integer beyond 53 bits has no exact +// float64, so it is not representable. +func pythonNumber(node *sitter.Node, source []byte) (float64, string) { + const dynamic, notRepresentable = "dynamic expression has no safe inverse", "number is not representable" + literal := node + sign := 1.0 + if node.Kind() == "unary_operator" { + operand := node.ChildByFieldName("argument") + if operand == nil || operand.Kind() != "integer" && operand.Kind() != "float" { + return 0, dynamic + } + text := code.Text(node, source) + if strings.HasPrefix(text, "-") { + sign = -1 + } else if !strings.HasPrefix(text, "+") { + return 0, dynamic + } + literal = operand + } + text := strings.ReplaceAll(code.Text(literal, source), "_", "") + if literal.Kind() == "integer" { + integer, err := strconv.ParseInt(strings.ToLower(text), 0, 64) + if err != nil || !adapter.ExactInteger(integer) { + return 0, notRepresentable + } + return sign * float64(integer), "" + } + value, err := strconv.ParseFloat(text, 64) + if err != nil || math.IsInf(value, 0) { + return 0, notRepresentable + } + return sign * value, "" +} + +func pythonDictionary(node *sitter.Node, source []byte, constants map[string]*adapter.Node) *adapter.Node { + start, end := int(node.StartByte()), int(node.EndByte()) + object := map[string]any{} + fields := map[string]*adapter.Node{} + complete := true + for _, item := range code.NamedChildren(node) { + // A splat or a comprehension has no static entry. + if item.Kind() != "pair" { + fields[code.Text(item, source)] = adapter.ReadOnly( + int(item.StartByte()), int(item.EndByte()), "dictionary entry is dynamic", + ) + complete = false + continue + } + keyNode := item.ChildByFieldName("key") + valueNode := item.ChildByFieldName("value") + // A write rewrites the dictionary as a literal with string keys, so a + // computed or non-string key makes it read-only. + text, ok := code.StringValue(pythonValue(keyNode, source, constants)) + if !ok { + text = code.Text(keyNode, source) + complete = false + } + value := pythonValue(valueNode, source, constants) + fields[text] = value + if code.Writable(value) { + object[text] = value.Value + } else { + complete = false + } + } + result := &adapter.Node{ + Value: object, Fields: fields, Start: start, End: end, Capability: adapter.CapabilityDirect, + Encode: code.PythonLiteral, + } + if !complete { + result.Value = nil + result.Capability = adapter.CapabilityReadOnly + result.Reason = "dictionary contains a dynamic entry" + result.Encode = nil + } + return result +} + +func pythonSequence(node *sitter.Node, source []byte, constants map[string]*adapter.Node) *adapter.Node { + start, end := int(node.StartByte()), int(node.EndByte()) + values := make([]any, 0, node.NamedChildCount()) + items := make([]*adapter.Node, 0, node.NamedChildCount()) + complete := true + for _, item := range code.NamedChildren(node) { + value := pythonValue(item, source, constants) + items = append(items, value) + values = append(values, value.Value) + if !code.Writable(value) { + complete = false + } + } + result := &adapter.Node{ + Value: values, Items: items, Collection: true, Start: start, End: end, Capability: adapter.CapabilityDirect, + Encode: code.PythonLiteral, + } + if node.Kind() == "tuple" { + // A tuple is written back as a tuple, so its type does not change. + result.Encode = code.PythonTuple + } + if !complete { + result.Value = nil + result.Capability = adapter.CapabilityReadOnly + result.Reason = "collection contains dynamic values" + result.Encode = nil + } + return result +} + +func pythonMessages(node *sitter.Node, source []byte, constants map[string]*adapter.Node) *adapter.Node { + node = pythonUnwrap(node) + if node != nil && node.Kind() == "identifier" { + start, end := int(node.StartByte()), int(node.EndByte()) + // A lambda parameter or a comprehension target is a value of the + // call, not the constant of the same name. + if pythonLocalBinding(node, code.Text(node, source), source) { + return adapter.ReadOnly(start, end, "dynamic expression has no safe inverse") + } + constant := pythonConstant(node, code.Text(node, source), start, source, constants) + if messages := code.MessageList(constant); messages != nil { + return messages + } + } + if node == nil || node.Kind() != "list" && node.Kind() != "tuple" { + if node == nil { + return adapter.ReadOnly(0, 0, "messages are missing") + } + return adapter.ReadOnly(int(node.StartByte()), int(node.EndByte()), "messages are not a literal list") + } + start, end := int(node.StartByte()), int(node.EndByte()) + var items []*adapter.Node + var messages []syncdomain.Message + complete := true + for _, child := range code.NamedChildren(node) { + item := pythonMessage(child, source, constants) + items = append(items, item) + role, roleOK := item.Fields["role"].Value.(string) + content, contentOK := item.Fields["content"].Value.(string) + if !roleOK || !contentOK || !code.Writable(item.Fields["content"]) { + complete = false + continue + } + role = code.NormalizeRole(role) + messages = append(messages, syncdomain.Message{Role: role, Content: content}) + } + result := &adapter.Node{ + Items: items, Collection: true, Start: start, End: end, Capability: adapter.CapabilityDirect, + Reason: "message structure is read-only", + } + if complete { + result.Value = messages + } else { + result.Capability = adapter.CapabilityReadOnly + result.Reason = "dynamic message content is read-only" + } + return result +} + +func pythonMessage(node *sitter.Node, source []byte, constants map[string]*adapter.Node) *adapter.Node { + fields := map[string]*adapter.Node{} + switch node.Kind() { + case "tuple", "list": + children := code.NamedChildren(node) + if len(children) == 2 { + fields["role"] = pythonValue(children[0], source, constants) + fields["content"] = pythonValue(children[1], source, constants) + } + case "dictionary": + dictionary := pythonDictionary(node, source, constants) + fields = dictionary.Fields + case "call": + function := code.Text(node.ChildByFieldName("function"), source) + role := "" + switch { + case strings.HasSuffix(function, "SystemMessage"): + role = syncdomain.RoleSystem + case strings.HasSuffix(function, "HumanMessage"): + role = syncdomain.RoleUser + case strings.HasSuffix(function, "AIMessage"): + role = syncdomain.RoleAssistant + } + if role != "" { + fields["role"] = &adapter.Node{Value: role} + arguments := pythonPositionalArguments(node) + if len(arguments) != 0 { + fields["content"] = pythonValue(arguments[0], source, constants) + } else if content := pythonKeywordFields(node, source, constants)["content"]; content != nil { + fields["content"] = content + } + } + } + if fields["role"] == nil { + fields["role"] = adapter.ReadOnly(int(node.StartByte()), int(node.EndByte()), "message role is not static") + } + if fields["content"] == nil { + fields["content"] = adapter.ReadOnly(int(node.StartByte()), int(node.EndByte()), "message content is not static") + } + return &adapter.Node{Fields: fields} +} + +func pythonDecoratedCollection( + node *sitter.Node, + aliases pythonAliases, + instances map[string]string, + source []byte, +) string { + kinds := aliases.at(node, source) + for _, child := range code.NamedChildren(node) { + if child.Kind() != "decorator" || child.NamedChildCount() == 0 { + continue + } + value := child.NamedChild(0) + if value.Kind() == "call" { + value = value.ChildByFieldName("function") + } + if value == nil { + continue + } + callable := code.Text(value, source) + if kind := kinds[callable]; kind == "tool-decorator" || kind == "kernel-tool-decorator" { + return "tools" + } + kind, method := code.Receiver(callable, func(receiver string) string { + return pythonInstanceKind(instances, node, receiver, source) + }) + switch kind { + case "fastmcp": + if method == "tool" { + return "tools" + } + if method == "prompt" { + return "prompts" + } + case "pydantic-agent": + if method == "tool" { + return "tools" + } + if method == "system_prompt" || method == "instructions" { + return "prompts" + } + } + } + return "" +} + +func encodePythonBool(value bool) ([]byte, error) { + if value { + return []byte("True"), nil + } + return []byte("False"), nil +} diff --git a/internal/sync/adapter/python/python_test.go b/internal/sync/adapter/python/python_test.go new file mode 100644 index 000000000..c520fa77c --- /dev/null +++ b/internal/sync/adapter/python/python_test.go @@ -0,0 +1,2694 @@ +package python + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sitter "github.com/tree-sitter/go-tree-sitter" + pythonlanguage "github.com/tree-sitter/tree-sitter-python/bindings/go" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/adapter" + "github.com/launchdarkly/ldcli/internal/sync/adapter/code" +) + +func TestPythonInspectorHandlesStaticAndDynamicLanguageForms(t *testing.T) { + source := []byte(` +from langchain_core.prompts import PromptTemplate, ChatPromptTemplate +from langchain_core.tools import tool +from fastmcp import FastMCP +from .values import TEXT, MISSING as missing + +TRUE = True +FALSE = False +NUMBER = 1.5 +TUPLE = ("system", TEXT) +OBJECT = {"text": TEXT, 1: "number", **{"extra": "value"}} +DYNAMIC = [TEXT, make_value()] +MUTATED = ["one"] +MUTATED.append("two") +MUTATED[0] = "changed" + +empty = PromptTemplate.from_template() +chat = ChatPromptTemplate.from_messages([("human", TEXT), ("ai", "Done"), {"role": "system"}]) + +@tool +def search(query: str): + """Search docs.""" + return query + +mcp = FastMCP("server") +@mcp.tool(description="Run a task") +def run_task(value: str): + """Run.""" + return value + +@mcp.prompt() +def dynamic_prompt(name: str): + """Prompt docs.""" + return f"Hello {name}" + +class Scope: + VALUE = "scoped" +`) + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: source}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte(`TEXT = "Hello {name}"`)}, + }, + }) + require.NoError(t, err) + require.NotNil(t, root) + assert.NotNil(t, root.Fields["tools"]) + assert.NotNil(t, root.Fields["prompts"]) + + dependencies, err := pythonAdapter{}.Dependencies(adapter.Document{Path: "src/main.py", Data: source}) + require.NoError(t, err) + assert.Contains(t, dependencies, "src/values.py") +} + +func TestPythonValueSafetyBranches(t *testing.T) { + document := adapter.Document{Path: "main.py", Data: []byte(` +TRUE = True +FALSE = False +OVERFLOW = 1e9999 +DICT = {"static": "value", "dynamic": make_value()} +LIST = ["value", make_value()] +message = AIMessage(content="done") +later = UNKNOWN +`)} + tree, err := code.ParseTree(document, sitter.NewLanguage(pythonlanguage.Language())) + require.NoError(t, err) + defer tree.Close() + + trueNode := code.FindNode(tree.RootNode(), document.Data, "true", "True") + require.NotNil(t, trueNode) + value := pythonValue(trueNode, document.Data, nil) + encoded, err := value.Encode(false) + require.NoError(t, err) + assert.Equal(t, "False", string(encoded)) + _, err = value.Encode("false") + require.Error(t, err) + + overflow := code.FindNode(tree.RootNode(), document.Data, "float", "1e9999") + require.NotNil(t, overflow) + assert.Equal(t, adapter.CapabilityReadOnly, pythonValue(overflow, document.Data, nil).Capability) + + dictionary := code.FindNode(tree.RootNode(), document.Data, "dictionary", `{"static": "value", "dynamic": make_value()}`) + require.NotNil(t, dictionary) + assert.Equal(t, adapter.CapabilityReadOnly, pythonDictionary(dictionary, document.Data, nil).Capability) + list := code.FindNode(tree.RootNode(), document.Data, "list", `["value", make_value()]`) + require.NotNil(t, list) + assert.Equal(t, adapter.CapabilityReadOnly, pythonSequence(list, document.Data, nil).Capability) + + call := code.FindNode(tree.RootNode(), document.Data, "call", `AIMessage(content="done")`) + require.NotNil(t, call) + message := pythonMessage(call, document.Data, nil) + assert.Equal(t, syncdomain.RoleAssistant, message.Fields["role"].Value) + assert.Equal(t, "done", message.Fields["content"].Value) + + identifier := code.FindNode(tree.RootNode(), document.Data, "identifier", "UNKNOWN") + require.NotNil(t, identifier) + wildcard := &adapter.Node{Reason: "unresolved", Code: syncdomain.ErrorImportUnresolved} + resolved := pythonValue(identifier, document.Data, map[string]*adapter.Node{"*": wildcard}) + assert.Contains(t, resolved.Reason, "star import") + + assert.Equal(t, "module.py", pythonModulePaths("main.py", "module")[0]) + assert.Empty(t, pythonMutatedName(nil, " ", nil, nil)) + assert.Equal(t, adapter.CapabilityReadOnly, pythonMessages(nil, nil, nil).Capability) +} + +func TestCodeParsersRejectUnsafeInput(t *testing.T) { + _, err := pythonAdapter{}.Dependencies(adapter.Document{Path: "main.py", Data: []byte{0xff}}) + require.Error(t, err) + _, err = pythonAdapter{}.Dependencies(adapter.Document{Path: "main.py", Data: make([]byte, code.MaxSourceBytes+1)}) + require.Error(t, err) + _, err = pythonAdapter{}.Dependencies(adapter.Document{Path: "main.py", Data: []byte("def broken(")}) + require.Error(t, err) + + result := code.MessageList(&adapter.Node{Items: []*adapter.Node{ + {Items: []*adapter.Node{{Value: "human"}, {Value: "Hello"}}}, + {Fields: map[string]*adapter.Node{"role": {Value: "system"}, "content": {Value: true}}}, + }}) + assert.Equal(t, adapter.CapabilityReadOnly, result.Capability) + assert.Nil(t, code.MessageList(nil)) + +} + +func TestPythonCollectionsTreatNoneAsStaticAndDynamicEntriesAsReadOnly(t *testing.T) { + document := adapter.Document{Path: "main.py", Data: []byte(` +NONE = [None, "text"] +SPLAT = {"a": "b", **extra} +NUMERIC = {1: "a"} +`)} + tree, err := code.ParseTree(document, sitter.NewLanguage(pythonlanguage.Language())) + require.NoError(t, err) + defer tree.Close() + + list := code.FindNode(tree.RootNode(), document.Data, "list", `[None, "text"]`) + require.NotNil(t, list) + value := pythonValue(list, document.Data, nil) + assert.Equal(t, adapter.CapabilityDirect, value.Capability) + assert.Equal(t, []any{nil, "text"}, value.Value) + encoded, err := value.Encode([]any{nil, "next"}) + require.NoError(t, err) + assert.Equal(t, `[None, "next"]`, string(encoded)) + + for _, source := range []string{`{"a": "b", **extra}`, `{1: "a"}`} { + dictionary := code.FindNode(tree.RootNode(), document.Data, "dictionary", source) + require.NotNil(t, dictionary, source) + result := pythonValue(dictionary, document.Data, nil) + assert.Equal(t, adapter.CapabilityReadOnly, result.Capability, source) + assert.Nil(t, result.Value, source) + } +} + +func TestMessageListsDoNotPanicOnMalformedItems(t *testing.T) { + for name, items := range map[string][]*adapter.Node{ + "scalar item": {{Value: 1.0}}, + "one-item pair": {{Items: []*adapter.Node{{Value: "human"}}}}, + "other fields": {{Fields: map[string]*adapter.Node{"text": {Value: "x"}}}}, + } { + t.Run(name, func(t *testing.T) { + result := code.MessageList(&adapter.Node{Items: items}) + require.NotNil(t, result) + assert.Equal(t, adapter.CapabilityReadOnly, result.Capability) + translated := code.TranslateFields( + map[string]*adapter.Node{"messages": {Items: items}}, code.BracePlaceholders, "messages", + )["messages"] + assert.Equal(t, adapter.CapabilityReadOnly, translated.Capability) + assert.Equal(t, "dynamic message content is read-only", translated.Reason) + }) + } +} + +func TestPythonImportedConstantsResolveRegardlessOfPosition(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte( + "from agents import Agent\nfrom .values import TEXT\nsupport = Agent(instructions=TEXT)\n", + )}, + Dependencies: map[string]adapter.Document{ + // The constant ends after the use in the primary file. + "src/values.py": {Path: "src/values.py", Data: []byte("# " + strings.Repeat("x", 300) + "\nTEXT = \"Hello\"\n")}, + }, + }) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1) + instructions := root.Fields["agents"].Items[0].Fields["instructions"] + assert.Equal(t, "Hello", instructions.Value) + assert.Equal(t, "src/values.py", instructions.File) +} + +func TestPythonImportedConstantsBindFromTheirStatementOn(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +early = Agent(instructions=TEXT) +from .values import TEXT +late = Agent(instructions=TEXT) + +def build(): + return Agent(instructions=TEXT) + +def local(): + before = Agent(instructions=LOCAL) + from .values import LOCAL + return before +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("TEXT = \"Hello\"\nLOCAL = \"Local\"\n")}, + }, + }) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + require.Len(t, agents, 4) + assert.Equal(t, adapter.CapabilityReadOnly, agents["early"].Fields["instructions"].Capability, "a use above the import is unbound") + assert.Equal(t, "Hello", agents["late"].Fields["instructions"].Value, "a use after the import resolves") + assert.Equal(t, "Hello", agents["build.Agent"].Fields["instructions"].Value, "a function body runs after the module imports") + assert.Equal(t, adapter.CapabilityReadOnly, agents["local.before"].Fields["instructions"].Capability, "a use above a local import is unbound") +} + +func TestPythonClassBodyLambdasAndComprehensionsSeeModuleNames(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" + +class Team: + TEXT = "class" + direct = Agent(instructions=TEXT) + build = lambda: Agent(instructions=TEXT) + names = [Agent(instructions=TEXT) for _ in range(1)] +`)}}) + require.NoError(t, err) + agents := map[string]string{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent.Fields["instructions"].Value.(string) + } + require.Len(t, agents, 3) + assert.Equal(t, "class", agents["Team.direct"], "class body code sees the class attribute") + for id, instructions := range agents { + if id != "Team.direct" { + assert.Equal(t, "module", instructions, "%s is in a scope of its own, which skips the class body", id) + } + } +} + +func TestPythonUnpackingAssignmentsBindTheirNames(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" +LIMIT = "kept" +ITEMS = ["a"] + +def build(pairs): + TEXT, other = pairs + return Agent(instructions=TEXT) + +plain = Agent(instructions=TEXT, name=LIMIT) +LIMIT, rest = values() +ITEMS[0], last = values() +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, adapter.CapabilityReadOnly, agents["build.Agent"].Fields["instructions"].Capability, "the unpacking binds a local that hides the module constant") + assert.Equal(t, "module", agents["plain"].Fields["instructions"].Value) + assert.Equal(t, adapter.CapabilityReadOnly, agents["plain"].Fields["name"].Capability, "a module unpacking rebinds the constant") + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityDirect, constants["TEXT"].Capability, "a local unpacking does not touch the module constant") + assert.Equal(t, adapter.CapabilityReadOnly, constants["LIMIT"].Capability) + assert.Equal(t, adapter.CapabilityReadOnly, constants["ITEMS"].Capability, "a subscript target in an unpacking mutates the constant") +} + +func TestPythonNamesImportedTwiceAreNotStatic(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +try: + from .values import TEXT +except ImportError: + from .fallback import TEXT +support = Agent(instructions=TEXT) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("TEXT = \"Values\"\n")}, + "src/fallback.py": {Path: "src/fallback.py", Data: []byte("TEXT = \"Fallback\"\n")}, + }, + }) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1) + instructions := root.Fields["agents"].Items[0].Fields["instructions"] + assert.Equal(t, adapter.CapabilityReadOnly, instructions.Capability, "two imports bind the name, so no one module is its value") + assert.Contains(t, instructions.Reason, "imported more than once") +} + +func TestPythonLaterLocalsHideOuterConstantsInTheWholeFunction(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" + +def build(): + prompt = TEXT + TEXT = "local" + return Agent(instructions=prompt) + +def unpack(pairs): + prompt = TEXT + TEXT, other = pairs + return Agent(instructions=prompt) + +plain = Agent(instructions=TEXT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.NotEqual(t, "module", agents["build.Agent"].Fields["instructions"].Value, "a later assignment makes the name local, so the earlier use is unbound") + assert.Equal(t, adapter.CapabilityReadOnly, agents["build.Agent"].Fields["instructions"].Capability) + assert.Equal(t, adapter.CapabilityReadOnly, agents["unpack.Agent"].Fields["instructions"].Capability, "a later unpacking makes the name local too") + assert.Equal(t, "module", agents["plain"].Fields["instructions"].Value) +} + +func TestPythonCollectionsOfMutatedImportsAreReadOnly(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from .values import ITEMS, TEXT +ITEMS.append("b") +WRAPPED = [ITEMS] +MAPPED = {"items": ITEMS} +PLAIN = [TEXT] +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("ITEMS = [\"a\"]\nTEXT = \"Hello\"\n")}, + }, + }) + require.NoError(t, err) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityReadOnly, constants["WRAPPED"].Capability, "a list that holds a mutated import is read-only") + assert.Equal(t, adapter.CapabilityReadOnly, constants["MAPPED"].Capability, "a dictionary that holds a mutated import is read-only") + assert.Equal(t, adapter.CapabilityDirect, constants["PLAIN"].Capability, "a list of a plain import stays writable") +} + +func TestPythonClassBodyImportsBindOnTheClassOnly(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from crewai import Agent + +class Team: + from agents import Agent + writer = Agent(instructions="class") + + def build(self): + return Agent(role="Writer", goal="Write", backstory="Years") +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + require.NotNil(t, agents["Team.writer"]) + assert.Equal(t, "class", agents["Team.writer"].Fields["instructions"].Value, "class body code sees the class import") + // The CrewAI agent of the method takes its role as its id. + require.NotNil(t, agents["Writer"], "a method resolves the name through the module, not the class") + assert.Equal(t, "Years", agents["Writer"].Fields["backstory"].Value) + assert.Nil(t, agents["Team.writer"].Fields["backstory"]) +} + +func TestPythonLoopTargetsShadowImportsFromTheirPointOn(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +from .values import TEXT +first = Agent(instructions=TEXT) +for TEXT in ["x"]: + pass +later = Agent(instructions=TEXT) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("TEXT = \"Hello\"\n")}, + }, + }) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, "Hello", agents["first"].Fields["instructions"].Value, "the import is visible before the loop rebinds the name") + assert.Equal(t, adapter.CapabilityReadOnly, agents["later"].Fields["instructions"].Capability, "the loop target hides the import from the loop on") +} + +func TestPythonFrameworkImportsWithAFallbackAreAmbiguous(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +try: + from agents import Agent +except ImportError: + from crewai import Agent +support = Agent(instructions="Help", backstory="Years") +`)}}) + require.NoError(t, err) + assert.Empty(t, root.Fields["agents"].Items, "either import can run, so the name has no one framework") + + // A try statement at the start of the file is a fallback too. + root, err = parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte( + "try:\n from agents import Agent\nexcept ImportError:\n from crewai import Agent\nsupport = Agent(instructions=\"Help\", backstory=\"Years\")\n", + )}}) + require.NoError(t, err) + assert.Empty(t, root.Fields["agents"].Items, "a try statement at byte zero is still a fallback") +} + +func TestPythonCasePatternsBindTheirCaptures(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" +LIMIT = "kept" + +def handle(event): + match event: + case [TEXT, *rest]: + return Agent(instructions=TEXT) + case Point(x=LIMIT) as other: + return Agent(instructions=LIMIT, name=other) + case Color.RED | {"key": value}: + return Agent(instructions=value) + return Agent(instructions=TEXT) + +match value: + case LIMIT: + pass +plain = Agent(instructions=TEXT, name=LIMIT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + for _, id := range []string{"handle.Agent", "handle.Agent#2", "handle.Agent#3", "handle.Agent#4"} { + require.NotNil(t, agents[id], id) + assert.Equal(t, adapter.CapabilityReadOnly, agents[id].Fields["instructions"].Capability, "%s reads a capture of the function", id) + } + assert.Equal(t, "module", agents["plain"].Fields["instructions"].Value) + assert.Equal(t, adapter.CapabilityReadOnly, agents["plain"].Fields["name"].Capability, "a module case pattern rebinds the constant") +} + +func TestPythonAliasesOfImportsFollowTheirPosition(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +from .values import TEXT +early = Agent(instructions=prompt) +prompt = TEXT +late = Agent(instructions=prompt) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("TEXT = \"Hello\"\n")}, + }, + }) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, adapter.CapabilityReadOnly, agents["early"].Fields["instructions"].Capability, "an alias of an import is not in scope before its assignment") + assert.Equal(t, "Hello", agents["late"].Fields["instructions"].Value) + assert.Equal(t, "src/values.py", agents["late"].Fields["instructions"].File) +} + +func TestPythonTypedSplatParametersHideOuterNames(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" + +def spread(*TEXT: str): + return Agent(instructions=TEXT) + +def keyed(**TEXT: int): + return Agent(instructions=TEXT) + +plain = Agent(instructions=TEXT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, adapter.CapabilityReadOnly, agents["spread.Agent"].Fields["instructions"].Capability, "a typed star parameter hides the constant") + assert.Equal(t, adapter.CapabilityReadOnly, agents["keyed.Agent"].Fields["instructions"].Capability, "a typed double star parameter hides the constant") + assert.Equal(t, "module", agents["plain"].Fields["instructions"].Value) +} + +func TestPythonReassignedResourcesGetOrdinalIDs(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent +from langchain_core.prompts import PromptTemplate + +agent = Agent(instructions="first") +agent = Agent(instructions="second") +prompt = PromptTemplate.from_template("one") +prompt = PromptTemplate.from_template("two") +single = Agent(instructions="only") +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + require.Len(t, agents, 3) + assert.Equal(t, "first", agents["agent#1"].Fields["instructions"].Value) + assert.Equal(t, "second", agents["agent#2"].Fields["instructions"].Value) + assert.True(t, adapter.IsPositionalID(agents["agent#1"]), "a reassigned name gives positional ids") + assert.Equal(t, "only", agents["single"].Fields["instructions"].Value) + assert.False(t, adapter.IsPositionalID(agents["single"]), "a name assigned once keeps its id") + prompts := map[string]*adapter.Node{} + for _, prompt := range root.Fields["prompts"].Items { + prompts[prompt.Fields["id"].Value.(string)] = prompt + } + assert.Equal(t, "one", prompts["prompt#1"].Fields["template"].Value) + assert.Equal(t, "two", prompts["prompt#2"].Fields["template"].Value) +} + +func TestPythonMutatedMessagesAreReadOnly(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from langchain_core.prompts import ChatPromptTemplate +from .values import MESSAGES +MESSAGES.append(("human", "more")) +chat = ChatPromptTemplate.from_messages(MESSAGES) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("MESSAGES = [(\"human\", \"Hi\")]\n")}, + }, + }) + require.NoError(t, err) + require.Len(t, root.Fields["prompts"].Items, 1) + messages := root.Fields["prompts"].Items[0].Fields["messages"] + assert.Equal(t, adapter.CapabilityReadOnly, messages.Capability, "a mutated import is not a static message list") + assert.Contains(t, messages.Reason, "mutated") +} + +func TestPythonParameterDefaultsRunInTheEnclosingScope(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" + +def make(agent=Agent(instructions=TEXT)): + pass + +def early(agent=Agent(instructions=LATER)): + pass + +LATER = "late" + +class Team: + TEXT = "class" + + def build(self, agent=Agent(instructions=TEXT)): + pass +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + require.Len(t, agents, 3) + assert.Equal(t, "module", agents["Agent"].Fields["instructions"].Value, "a default of a module function reads the module constant, in module scope") + assert.Equal(t, adapter.CapabilityReadOnly, agents["Agent#2"].Fields["instructions"].Capability, "a default runs with the definition, before a later constant") + assert.Equal(t, "class", agents["Team.Agent"].Fields["instructions"].Value, "a default of a method reads the class attribute") +} + +func TestPythonClientResetsDoNotDropTheClientKind(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from openai import OpenAI + +class Assistant: + def __init__(self): + self.client = OpenAI() + + def ask(self): + return self.client.chat.completions.create(model="gpt-5", messages=[{"role": "user", "content": "Hi"}]) + + def close(self): + self.client = None +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["prompts"].Items, 1, "a reset to None does not change the kind of the client") +} + +func TestPythonWalrusBindingsNameResourcesAndClients(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent +from openai import OpenAI + +if (agent := Agent(instructions="walrus")): + pass +if (client := OpenAI()): + reply = client.chat.completions.create(model="gpt-5", messages=[{"role": "user", "content": "Hi"}]) +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1, "a walrus names its resource once") + assert.Equal(t, "agent", root.Fields["agents"].Items[0].Fields["id"].Value, "the walrus variable names the agent") + require.Len(t, root.Fields["prompts"].Items, 1, "a walrus client is a client") + assert.Equal(t, "reply", root.Fields["prompts"].Items[0].Fields["id"].Value) + + nested, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent +team = [first := Agent(instructions="first"), Agent(instructions="second")] +`)}}) + require.NoError(t, err) + agents := map[string]string{} + for _, agent := range nested.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent.Fields["instructions"].Value.(string) + } + assert.Equal(t, map[string]string{"first": "first", "Agent": "second"}, agents, "a walrus inside a list names its call, and the list is not a chain") +} + +func TestPythonLambdaDefaultsRunAtDefinition(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" +early = lambda agent=Agent(instructions=LATER): agent +LATER = "late" +ready = lambda agent=Agent(instructions=TEXT): agent +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + require.Len(t, agents, 2) + assert.Equal(t, adapter.CapabilityReadOnly, agents["Agent"].Fields["instructions"].Capability, "a lambda default runs with the definition, before a later constant") + assert.Equal(t, "module", agents["Agent#2"].Fields["instructions"].Value) +} + +func TestPythonClassNameMutationsFreezeTheClassAttribute(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +class Bot: + items = ["a"] + names = ["n"] + kept = "same" + +Bot.items.append("b") +Bot.names[0] = "x" +`)}}) + require.NoError(t, err) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityReadOnly, constants["Bot.items"].Capability, "a mutating call through the class name freezes the attribute") + assert.Equal(t, adapter.CapabilityReadOnly, constants["Bot.names"].Capability, "a subscript write through the class name freezes the attribute") + assert.Equal(t, adapter.CapabilityDirect, constants["Bot.kept"].Capability) +} + +func TestPythonWalrusBindsALocalVariable(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" + +def build(items): + if (TEXT := items[0]): + pass + return Agent(instructions=TEXT) + +plain = Agent(instructions=TEXT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, adapter.CapabilityReadOnly, agents["build.Agent"].Fields["instructions"].Capability, "the walrus binds a local that hides the module constant") + assert.Equal(t, "module", agents["plain"].Fields["instructions"].Value) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityDirect, constants["TEXT"].Capability, "the walrus in the function does not mutate the module constant") +} + +func TestPythonScopesResolveUsesAndMutationsAlike(t *testing.T) { + bundle := adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +from .values import TEXT + +LIST = ["one"] +MESSAGES = [("human", TEXT)] + +def build(): + prompt = "local" + LIST.append("two") + return Agent(instructions=prompt) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte(`TEXT = "Hello"`)}, + }, + } + root, err := parsePythonBundle(bundle) + require.NoError(t, err) + + require.Len(t, root.Fields["agents"].Items, 1) + assert.Equal(t, "local", root.Fields["agents"].Items[0].Fields["instructions"].Value, + "a use resolves the constant of its own function") + var list, messages *adapter.Node + for _, constant := range root.Fields["constants"].Items { + switch constant.Fields["id"].Value { + case "LIST": + list = constant + case "MESSAGES": + messages = constant + } + } + require.NotNil(t, list) + assert.Equal(t, adapter.CapabilityReadOnly, list.Capability, "a mutation inside a function marks the module-level list") + require.NotNil(t, messages) + assert.Equal(t, adapter.CapabilityDirect, messages.Capability, "an imported value inside a constant resolves") +} + +func TestPythonReturnedPromptCallsAreCollected(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from langchain_core.prompts import PromptTemplate + +def build(): + return PromptTemplate.from_template("Hello {name}") +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["prompts"].Items, 1) + prompt := root.Fields["prompts"].Items[0] + assert.Equal(t, "build.PromptTemplate.from_template", prompt.Fields["id"].Value) + assert.Equal(t, "Hello {{name}}", prompt.Fields["template"].Value) +} + +func TestPythonPromptsOnSubscriptAndUnpackingTargetsAreCollected(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from langchain_core.prompts import PromptTemplate + +PROMPTS = {} + +def build(): + PROMPTS["greet"] = PromptTemplate.from_template("Hi {name}") + first, second = PromptTemplate.from_template("One"), PromptTemplate.from_template("Two") + PROMPTS["named"] = named = PromptTemplate.from_template("Named") + return first, second +`)}}) + require.NoError(t, err) + templates := map[string]string{} + for _, prompt := range root.Fields["prompts"].Items { + templates[prompt.Fields["id"].Value.(string)] = prompt.Fields["template"].Value.(string) + } + assert.Equal(t, map[string]string{ + "build.PromptTemplate.from_template": "Hi {{name}}", + "build.PromptTemplate.from_template#2": "One", + "build.PromptTemplate.from_template#3": "Two", + "build.named": "Named", + }, templates, "a prompt without a variable name takes the name of the call, once per prompt") + for _, prompt := range root.Fields["prompts"].Items { + if id := prompt.Fields["id"].Value.(string); id != "build.named" { + assert.True(t, adapter.IsPositionalID(prompt), "%s is positional", id) + } + } +} + +func TestPythonMutationOfACollectionLeavesTheImportedValueWritable(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +from .values import TEXT + +LIST = [TEXT] +LIST.append("more") +support = Agent(instructions=TEXT) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte(`TEXT = "Hello"`)}, + }, + }) + require.NoError(t, err) + instructions := root.Fields["agents"].Items[0].Fields["instructions"] + assert.Equal(t, adapter.CapabilityDirect, instructions.Capability, "the mutated list does not mark the import") + assert.NotNil(t, instructions.Encode) +} + +func TestPythonConstantsUseEarlierLocalConstants(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +SYSTEM = "You help." +MESSAGES = [("system", SYSTEM), ("human", "Hi")] +LATER = [FUTURE] +FUTURE = "late" +`)}}) + require.NoError(t, err) + byID := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + byID[constant.Fields["id"].Value.(string)] = constant + } + messages := code.MessageList(byID["MESSAGES"]) + require.NotNil(t, messages) + assert.Equal(t, adapter.CapabilityDirect, messages.Capability, "a local system prompt resolves") + assert.Equal(t, adapter.CapabilityReadOnly, byID["LATER"].Capability, "a constant declared later is not in scope") +} + +func TestPythonMessagesResolveTheScopedConstant(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from langchain_core.prompts import ChatPromptTemplate + +MESSAGES = [("human", "module")] + +def build(): + MESSAGES = [("human", "local")] + return ChatPromptTemplate.from_messages(MESSAGES) + +def early(): + chat = ChatPromptTemplate.from_messages(MESSAGES_LATER) + MESSAGES_LATER = [("human", "late")] + return chat +`)}}) + require.NoError(t, err) + byID := map[string]*adapter.Node{} + for _, prompt := range root.Fields["prompts"].Items { + byID[prompt.Fields["id"].Value.(string)] = prompt + } + local := byID["build.ChatPromptTemplate.from_messages"] + require.NotNil(t, local) + assert.Equal(t, []syncdomain.Message{{Role: syncdomain.RoleUser, Content: "local"}}, local.Fields["messages"].Value) + early := byID["early.chat"] + require.NotNil(t, early) + assert.Equal(t, adapter.CapabilityReadOnly, early.Fields["messages"].Capability, "a name declared after the call is not in scope") +} + +func TestPythonAllowlistMatchesOnlyTheExactModule(t *testing.T) { + assert.NotNil(t, pythonAllowedImports("agents")) + assert.Nil(t, pythonAllowedImports("agents.prompts"), "a repository package with a framework prefix is followed as an import") + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte( + "from agents import Agent\nfrom agents.prompts import TEXT\nsupport = Agent(instructions=TEXT)\n", + )}, + Dependencies: map[string]adapter.Document{ + // An absolute module resolves from the repository root. + "agents/prompts.py": {Path: "agents/prompts.py", Data: []byte(`TEXT = "Hello"`)}, + }, + }) + require.NoError(t, err) + assert.Equal(t, "Hello", root.Fields["agents"].Items[0].Fields["instructions"].Value) +} + +func TestPythonParenthesizedAndConcatenatedStringsAreStatic(t *testing.T) { + document := adapter.Document{Path: "main.py", Data: []byte("TEXT = (\"one \"\n 'two')\nSOLO = (\"solo\")\nMIXED = (\"a\" f\"{b}\")\n")} + tree, err := code.ParseTree(document, sitter.NewLanguage(pythonlanguage.Language())) + require.NoError(t, err) + defer tree.Close() + + joined := code.FindNode(tree.RootNode(), document.Data, "parenthesized_expression", "(\"one \"\n 'two')") + require.NotNil(t, joined) + value := pythonValue(joined, document.Data, nil) + assert.Equal(t, "one two", value.Value) + assert.Equal(t, adapter.CapabilityDirect, value.Capability) + encoded, err := value.Encode("next") + require.NoError(t, err) + assert.Equal(t, `"next"`, string(encoded), "the first quote style wins") + assert.Equal(t, len("(\"one \"\n 'two')")-2, value.End-value.Start, "the edit replaces every part") + + solo := code.FindNode(tree.RootNode(), document.Data, "parenthesized_expression", `("solo")`) + require.NotNil(t, solo) + assert.Equal(t, "solo", pythonValue(solo, document.Data, nil).Value) + mixed := code.FindNode(tree.RootNode(), document.Data, "concatenated_string", "\"a\" f\"{b}\"") + require.NotNil(t, mixed) + assert.Equal(t, adapter.CapabilityReadOnly, pythonValue(mixed, document.Data, nil).Capability) +} + +func TestPythonLocalAssignmentShadowsAnImport(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte( + "from agents import Agent\nfrom .values import TEXT\nTEXT = \"local\"\nsupport = Agent(instructions=TEXT)\n", + )}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte(`TEXT = "imported"`)}, + }, + }) + require.NoError(t, err) + assert.Equal(t, "local", root.Fields["agents"].Items[0].Fields["instructions"].Value) +} + +func TestPythonImportsReadTheSyntaxTree(t *testing.T) { + document := adapter.Document{Path: "main.py", Data: []byte( + "from agents import Agent as Bot, function_tool # tools\nfrom .values import (TEXT as text,\n OTHER)\n", + )} + tree, err := code.ParseTree(document, sitter.NewLanguage(pythonlanguage.Language())) + require.NoError(t, err) + defer tree.Close() + + last := tree.RootNode().NamedChild(tree.RootNode().NamedChildCount() - 1) + assert.Equal(t, map[string]string{"Bot": "openai-agent", "function_tool": "tool-decorator"}, pythonImports(tree.RootNode(), document.Data).at(last, document.Data)) + assert.Equal(t, []pythonImportBinding{ + {module: ".values", name: "TEXT", alias: "text"}, + {module: ".values", name: "OTHER", alias: "OTHER"}, + }, pythonImportBindings(tree.RootNode(), document.Data)) +} + +func TestNoneIsAConstant(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte("NOTHING = None\n")}}) + require.NoError(t, err) + require.Len(t, root.Fields["constants"].Items, 1) + assert.Equal(t, "NOTHING", root.Fields["constants"].Items[0].Fields["id"].Value) +} + +func TestPythonNestedConstructorsAreResources(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from crewai import Agent, Task +agents = [Agent(role="writer", goal="Write"), Agent(role="editor", goal="Edit")] +task = Task(description="Draft", agent=Agent(role="helper", goal="Help")) +`)}}) + require.NoError(t, err) + ids := []string{} + for _, agent := range root.Fields["agents"].Items { + ids = append(ids, agent.Fields["id"].Value.(string)) + } + assert.Len(t, ids, 3, "each nested constructor is a resource") + assert.Len(t, root.Fields["prompts"].Items, 1, "the assigned task is named once") +} + +func TestPythonStringsKeepTheLineEndingsOfTheFile(t *testing.T) { + document := adapter.Document{Path: "main.py", Data: []byte("TEXT = \"\"\"first\r\nsecond\"\"\"\r\n")} + tree, err := code.ParseTree(document, sitter.NewLanguage(pythonlanguage.Language())) + require.NoError(t, err) + defer tree.Close() + + literal := code.FindNode(tree.RootNode(), document.Data, "string", "\"\"\"first\r\nsecond\"\"\"") + require.NotNil(t, literal) + value := pythonValue(literal, document.Data, nil) + assert.Equal(t, "first\nsecond", value.Value, "the value has one line ending") + encoded, err := value.Encode("one\ntwo\nthree") + require.NoError(t, err) + assert.Equal(t, "\"\"\"one\r\ntwo\r\nthree\"\"\"", string(encoded), "the write keeps the line ending of the file") +} + +func TestPythonInstancesResolveByScope(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from openai import OpenAI +from anthropic import Anthropic + +client = OpenAI() +first = client.chat.completions.create(model="gpt", messages=[{"role": "user", "content": "one"}]) + +def other(): + client = Anthropic() + return client.messages.create(model="claude", messages=[{"role": "user", "content": "two"}]) + +shared = OpenAI() +shared = Anthropic() +third = shared.messages.create(model="claude", messages=[{"role": "user", "content": "three"}]) +`)}}) + require.NoError(t, err) + ids := map[string]bool{} + for _, prompt := range root.Fields["prompts"].Items { + ids[prompt.Fields["id"].Value.(string)] = true + } + assert.True(t, ids["first"], "the module-level client is an OpenAI client") + assert.True(t, ids["other.client.messages.create"], "the local client is an Anthropic client") + assert.False(t, ids["third"], "a name that two frameworks assign is ambiguous") +} + +func TestPythonOpenAIAgentInstructionsArePlainText(t *testing.T) { + for name, text := range map[string]string{ + "placeholder-like": "Return {answer} only.", + "JSON example": `Reply as JSON: {"answer": "yes"}`, + "double braces": "Keep {{literal}} braces.", + } { + t.Run(name, func(t *testing.T) { + source := "from agents import Agent\nsupport = Agent(instructions=" + string(code.EncodePythonString(text, `"`)) + ")\n" + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(source)}}) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1) + instructions := root.Fields["agents"].Items[0].Fields["instructions"] + + assert.Equal(t, text, instructions.Value, "the Agents SDK sends instructions without formatting them") + require.Equal(t, adapter.CapabilityDirect, instructions.Capability, instructions.Reason) + encoded, err := instructions.Encode(text + " Be brief.") + require.NoError(t, err) + assert.Equal(t, string(code.EncodePythonString(text+" Be brief.", `"`)), string(encoded)) + }) + } +} + +func TestPythonCrewAIAgentTextTranslatesPlaceholders(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from crewai import Agent + +researcher = Agent(role="Researcher", goal="Research {topic}", backstory="Expert in {topic}.") +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1) + goal := root.Fields["agents"].Items[0].Fields["goal"] + assert.Equal(t, "Research {{topic}}", goal.Value, "CrewAI fills {topic} from kickoff inputs") + encoded, err := goal.Encode("Study {{topic}}") + require.NoError(t, err) + assert.Equal(t, `"Study {topic}"`, string(encoded)) +} + +func TestPythonHTMLInPromptsRoundTrips(t *testing.T) { + for name, html := range map[string]string{ + "tags": "

Answer in bold.

", + "attributes": `docs`, + "entities": "Use <tags> & "quotes".", + "multiline": "", + "inline style": `

Warn

`, + "style block": "

Warn

", + "script block": "", + "template vars": "

Hello {name}

", + } { + t.Run(name, func(t *testing.T) { + for _, quote := range []string{`"`, `"""`} { + source := "from agents import Agent\nsupport = Agent(instructions=" + string(code.EncodePythonString(html, quote)) + ")\n" + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(source)}}) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1) + instructions := root.Fields["agents"].Items[0].Fields["instructions"] + + assert.Equal(t, html, instructions.Value, "quote %s", quote) + require.Equal(t, adapter.CapabilityDirect, instructions.Capability, "quote %s: %s", quote, instructions.Reason) + encoded, err := instructions.Encode(html + "
") + require.NoError(t, err) + decoded, _, err := code.DecodePythonString(encoded) + require.NoError(t, err) + assert.Equal(t, html+"
", decoded, "quote %s", quote) + } + }) + } +} + +func TestPythonModuleQualifiedFrameworkCallsAreFound(t *testing.T) { + for name, source := range map[string]string{ + "import module": "import agents\nsupport = agents.Agent(name=\"Support\", instructions=\"Help.\")\n", + "import alias": "import agents as oa\nsupport = oa.Agent(name=\"Support\", instructions=\"Help.\")\n", + "openai qualified": "import openai\nclient = openai.OpenAI()\nreply = client.chat.completions.create(model=\"gpt-5\", messages=[{\"role\": \"system\", \"content\": \"Help.\"}])\n", + } { + t.Run(name, func(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(source)}}) + require.NoError(t, err) + found := len(root.Fields["agents"].Items) + len(root.Fields["prompts"].Items) + assert.Equal(t, 1, found, "the framework call is found") + }) + } +} + +func TestPythonDynamicLocalShadowsAnImport(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte( + "from agents import Agent\nfrom .values import TEXT\nTEXT = load()\nsupport = Agent(instructions=TEXT)\n", + )}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte(`TEXT = "imported"`)}, + }, + }) + require.NoError(t, err) + instructions := root.Fields["agents"].Items[0].Fields["instructions"] + assert.Equal(t, adapter.CapabilityReadOnly, instructions.Capability, "the dynamic local hides the import") + assert.Nil(t, instructions.Value) +} + +func TestPythonUnnamedNestedConstructorsGetDistinctIDs(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from pydantic_ai import Agent +from crewai import Agent as Crew +team = [Agent("model-a"), Agent("model-b")] +crew = [Crew(role="writer"), Crew(role="editor")] +`)}}) + require.NoError(t, err) + ids := []string{} + for _, agent := range root.Fields["agents"].Items { + ids = append(ids, agent.Fields["id"].Value.(string)) + } + assert.ElementsMatch(t, []string{"Agent", "Agent#2", "writer", "editor"}, ids) +} + +func TestPythonAttributeAssignmentsKeepFrameworkResources(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +class Service: + def __init__(self): + self.agent = Agent(name="support", instructions="Help") +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1) + assert.Equal(t, "support", root.Fields["agents"].Items[0].Fields["id"].Value) +} + +func TestPythonKeywordOnlyPromptFactoriesHaveFields(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from langchain_core.prompts import PromptTemplate, ChatPromptTemplate +single = PromptTemplate.from_template(template="Hello {name}") +chat = ChatPromptTemplate.from_messages(messages=[("human", "Hi")]) +`)}}) + require.NoError(t, err) + prompts := map[string]*adapter.Node{} + for _, prompt := range root.Fields["prompts"].Items { + prompts[prompt.Fields["id"].Value.(string)] = prompt + } + assert.Equal(t, "Hello {{name}}", prompts["single"].Fields["template"].Value) + assert.Equal(t, []syncdomain.Message{{Role: syncdomain.RoleUser, Content: "Hi"}}, prompts["chat"].Fields["messages"].Value) +} + +func TestPythonNumbersWithSignsSeparatorsAndBases(t *testing.T) { + document := adapter.Document{Path: "main.py", Data: []byte("A = -1\nB = 1_000\nC = 0x10\nD = 0o17\nE = 0b101\nF = -2.5\nG = -x\n")} + tree, err := code.ParseTree(document, sitter.NewLanguage(pythonlanguage.Language())) + require.NoError(t, err) + defer tree.Close() + for text, expected := range map[string]float64{"-1": -1, "1_000": 1000, "0x10": 16, "0o17": 15, "0b101": 5, "-2.5": -2.5} { + kind := "integer" + if strings.HasPrefix(text, "-") { + kind = "unary_operator" + } + node := code.FindNode(tree.RootNode(), document.Data, kind, text) + require.NotNil(t, node, text) + value := pythonValue(node, document.Data, nil) + assert.Equal(t, expected, value.Value, text) + assert.Equal(t, adapter.CapabilityDirect, value.Capability, text) + } + dynamic := code.FindNode(tree.RootNode(), document.Data, "unary_operator", "-x") + require.NotNil(t, dynamic) + assert.Equal(t, adapter.CapabilityReadOnly, pythonValue(dynamic, document.Data, nil).Capability) +} + +func TestPythonIntegersBeyondExactFloatsAreReadOnly(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +EXACT = 9007199254740992 +BIG = 9007199254740993 +support = Agent(instructions="Help", exact=EXACT, big=9007199254740993, huge=99999999999999999999, negative=-9007199254740993, floating=1e20) +`)}}) + require.NoError(t, err) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, float64(1<<53), constants["EXACT"].Value, "the largest exact integer is a constant") + assert.Nil(t, constants["BIG"], "an integer without an exact float64 is not a literal constant") + require.Len(t, root.Fields["agents"].Items, 1) + fields := root.Fields["agents"].Items[0].Fields + assert.Equal(t, float64(1<<53), fields["exact"].Value) + for _, field := range []string{"big", "huge", "negative"} { + assert.Equal(t, adapter.CapabilityReadOnly, fields[field].Capability, "%s has no exact float64", field) + assert.Contains(t, fields[field].Reason, "number is not representable", field) + } + assert.Equal(t, 1e20, fields["floating"].Value, "a float literal keeps its float value") +} + +func TestPythonConstantAliasChainsResolve(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +BASE = "You help." +ALIAS = BASE +MESSAGES = [("system", ALIAS)] +`)}}) + require.NoError(t, err) + for _, constant := range root.Fields["constants"].Items { + if constant.Fields["id"].Value == "MESSAGES" { + messages := code.MessageList(constant) + require.NotNil(t, messages) + assert.Equal(t, adapter.CapabilityDirect, messages.Capability, "a two-hop alias resolves") + } + } +} + +func TestPythonListedPromptsAndModuleQualifiedFactories(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +import langchain_core.prompts as p +from langchain_core.prompts import PromptTemplate +prompts = [PromptTemplate.from_template("one"), PromptTemplate.from_template("two")] +chat = p.ChatPromptTemplate.from_messages([("human", "Hi")]) +`)}}) + require.NoError(t, err) + ids := map[string]*adapter.Node{} + for _, prompt := range root.Fields["prompts"].Items { + ids[prompt.Fields["id"].Value.(string)] = prompt + } + assert.Equal(t, "one", ids["prompts#1"].Fields["template"].Value) + assert.Equal(t, "two", ids["prompts#2"].Fields["template"].Value) + require.NotNil(t, ids["chat"], "a module-qualified factory is a prompt") + assert.Equal(t, []syncdomain.Message{{Role: syncdomain.RoleUser, Content: "Hi"}}, ids["chat"].Fields["messages"].Value) +} + +func TestPythonQualifiedIDsIncludeEveryEnclosingScope(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from openai import OpenAI +from anthropic import Anthropic + +class First: + def setup(self): + client = OpenAI() + prompt = "first" + return client.chat.completions.create(model="gpt", messages=[{"role": "user", "content": prompt}]) + +class Second: + def setup(self): + client = Anthropic() + prompt = "second" + return client.messages.create(model="claude", messages=[{"role": "user", "content": prompt}]) + +def outer(): + SHARED = "outer" + def inner(): + return SHARED +`)}}) + require.NoError(t, err) + ids := map[string]*adapter.Node{} + for _, prompt := range root.Fields["prompts"].Items { + ids[prompt.Fields["id"].Value.(string)] = prompt + } + require.NotNil(t, ids["First.setup.client.chat.completions.create"], "the class is part of the id") + require.NotNil(t, ids["Second.setup.client.messages.create"], "each class keeps its own client kind") + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + require.NotNil(t, constants["First.setup.prompt"]) + require.NotNil(t, constants["Second.setup.prompt"]) + require.NotNil(t, constants["outer.SHARED"]) +} + +func TestPythonChainedAssignmentsAndTaskOutputs(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent +from crewai import Task +SYSTEM = DEFAULT = "You help." +support = Agent(instructions=SYSTEM) +task = Task(description="Write about {topic}", expected_output="A post on {topic}") +`)}}) + require.NoError(t, err) + assert.Equal(t, "You help.", root.Fields["agents"].Items[0].Fields["instructions"].Value, "a chained assignment is static") + require.Len(t, root.Fields["prompts"].Items, 1) + task := root.Fields["prompts"].Items[0] + assert.Equal(t, "A post on {{topic}}", task.Fields["expected_output"].Value, "the task output translates its placeholders") + encoded, err := task.Fields["expected_output"].Encode("An essay on {{topic}}") + require.NoError(t, err) + assert.Equal(t, `"An essay on {topic}"`, string(encoded)) +} + +func TestPlaceholderResidueBesideAVariableIsReadOnly(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from langchain_core.prompts import PromptTemplate +escaped = PromptTemplate.from_template("Use }} before {name}") +plain = PromptTemplate.from_template("Hello {name}") +`)}}) + require.NoError(t, err) + prompts := map[string]*adapter.Node{} + for _, prompt := range root.Fields["prompts"].Items { + prompts[prompt.Fields["id"].Value.(string)] = prompt + } + assert.Equal(t, adapter.CapabilityReadOnly, prompts["escaped"].Fields["template"].Capability, "a stray brace before the placeholder is ambiguous") + assert.Equal(t, "Hello {{name}}", prompts["plain"].Fields["template"].Value) +} + +func TestPythonDecoratedIDsIncludeEveryEnclosingScope(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from mcp.server.fastmcp import FastMCP +mcp = FastMCP("server") + +class First: + @mcp.tool() + def run(self, value: str): + """First.""" + return value + +class Second: + @mcp.tool() + def run(self, value: str): + """Second.""" + return value + + @mcp.prompt() + def greet(self, name: str): + """Greet.""" + return name +`)}}) + require.NoError(t, err) + tools := map[string]*adapter.Node{} + for _, tool := range root.Fields["tools"].Items { + tools[tool.Fields["id"].Value.(string)] = tool + } + require.Len(t, tools, 2, "two decorated methods with one name keep distinct ids") + assert.Equal(t, "First.", tools["First.run"].Fields["description"].Value) + assert.Equal(t, "Second.", tools["Second.run"].Fields["description"].Value) + require.Len(t, root.Fields["prompts"].Items, 1) + assert.Equal(t, "Second.greet", root.Fields["prompts"].Items[0].Fields["id"].Value) +} + +func TestPythonAttributeClientsAreRequests(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from openai import OpenAI +from anthropic import Anthropic + +class Assistant: + def __init__(self): + self.client = OpenAI() + + def ask(self): + return self.client.chat.completions.create(model="gpt", messages=[{"role": "user", "content": "one"}]) + +class Other: + def __init__(self): + self.client = Anthropic() + + def ask(self): + return self.client.messages.create(model="claude", messages=[{"role": "user", "content": "two"}]) +`)}}) + require.NoError(t, err) + prompts := map[string]*adapter.Node{} + for _, prompt := range root.Fields["prompts"].Items { + prompts[prompt.Fields["id"].Value.(string)] = prompt + } + require.Len(t, prompts, 2, "a client stored on an attribute keeps its kind") + require.Contains(t, prompts, "Assistant.ask.self.client.chat.completions.create") + assert.Equal(t, "gpt", prompts["Assistant.ask.self.client.chat.completions.create"].Fields["model"].Value) + require.Contains(t, prompts, "Other.ask.self.client.messages.create") + assert.Equal(t, "claude", prompts["Other.ask.self.client.messages.create"].Fields["model"].Value) +} + +func TestPythonMutatedImportIsReadOnly(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +from .values import TEXT, OTHER +TEXT += " now" +support = Agent(instructions=TEXT) +other = Agent(instructions=OTHER) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("TEXT = \"Help\"\nOTHER = \"Keep\"\n")}, + }, + }) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + mutated := agents["support"].Fields["instructions"] + assert.Equal(t, adapter.CapabilityReadOnly, mutated.Capability, "an import that this file mutates is read-only") + assert.Equal(t, "mutated constants are read-only", mutated.Reason) + assert.Equal(t, adapter.CapabilityDirect, agents["other"].Fields["instructions"].Capability) +} + +func TestPythonStarImportIsFollowed(t *testing.T) { + source := []byte(` +from agents import Agent +from .values import * +support = Agent(instructions=TEXT) +`) + dependencies, err := pythonAdapter{}.Dependencies(adapter.Document{Path: "src/main.py", Data: source}) + require.NoError(t, err) + assert.Contains(t, dependencies, "src/values.py", "a star import names its module as a dependency") + + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: source}, + Dependencies: map[string]adapter.Document{"src/values.py": {Path: "src/values.py", Data: []byte("TEXT = \"Help\"\n")}}, + }) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1) + instructions := root.Fields["agents"].Items[0].Fields["instructions"] + assert.Equal(t, adapter.CapabilityReadOnly, instructions.Capability) + assert.Contains(t, instructions.Reason, "star import cannot resolve TEXT statically") +} + +func TestPythonChainedAttributeAssignmentsNameTheVariable(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +class Bot: + def __init__(self): + self.agent = local = Agent(instructions="Local") + agent = self.backup = Agent(instructions="Backup") + self.only = Agent(instructions="Attribute") +`)}}) + require.NoError(t, err) + agents := map[string]string{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent.Fields["instructions"].Value.(string) + } + assert.Equal(t, map[string]string{ + "Bot.__init__.local": "Local", + "Bot.__init__.agent": "Backup", + "Bot.__init__.Agent": "Attribute", + }, agents, "a chain names its call once, after its variable, and a chain of attributes names it like an unassigned call") +} + +func TestPythonLambdasSeeLaterImportsAndConstants(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +build = lambda: Agent(instructions=TEXT) +early = Agent(instructions=TEXT) +from agents import Agent +TEXT = "Later" +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1, "the lambda body runs after the module loads, and the immediate call does not see the import") + assert.Equal(t, "Later", root.Fields["agents"].Items[0].Fields["instructions"].Value) +} + +func TestPythonLoopWithAndExceptTargetsHideConstants(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" +LIMIT = "kept" + +def loop(items): + for TEXT in items: + pass + return Agent(instructions=TEXT) + +def context(): + with open("f") as TEXT: + return Agent(instructions=TEXT) + +def catch(): + try: + pass + except ValueError as TEXT: + return Agent(instructions=TEXT) + +def unpack(pairs): + for key, TEXT in pairs: + pass + return Agent(instructions=TEXT) + +names = [TEXT for TEXT in ["a"]] +plain = Agent(instructions=TEXT, name=LIMIT) +for LIMIT in ["b"]: + pass +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + for _, id := range []string{"loop.Agent", "context.Agent", "catch.Agent", "unpack.Agent"} { + require.NotNil(t, agents[id], id) + assert.Equal(t, adapter.CapabilityReadOnly, agents[id].Fields["instructions"].Capability, "%s reads the local binding", id) + } + assert.Equal(t, "module", agents["plain"].Fields["instructions"].Value, "a comprehension target binds its name in the comprehension only") + assert.Equal(t, adapter.CapabilityReadOnly, agents["plain"].Fields["name"].Capability, "a module loop rebinds the constant") +} + +func TestPythonAttributeAssignedPromptIsCollected(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from langchain_core.prompts import PromptTemplate + +class Assistant: + def __init__(self): + self.template = PromptTemplate.from_template("Hi {name}") +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["prompts"].Items, 1, "a prompt on an attribute is collected once") + prompt := root.Fields["prompts"].Items[0] + assert.Equal(t, "Assistant.__init__.self.template", prompt.Fields["id"].Value) + assert.Equal(t, "Hi {{name}}", prompt.Fields["template"].Value) +} + +func TestPythonSplatArgumentsFillNoPositionalField(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent +support = Agent(name="support", instructions="Help", **options) +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1) + agent := root.Fields["agents"].Items[0] + assert.Equal(t, "support", agent.Fields["name"].Value, "a splat does not replace the keyword name") + assert.Equal(t, "Help", agent.Fields["instructions"].Value) + assert.Equal(t, adapter.CapabilityDirect, agent.Fields["name"].Capability) +} + +func TestPythonMethodsDoNotSeeClassAttributes(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent +SYSTEM = "module" + +class Bot: + SYSTEM = "class" + direct = Agent(instructions=SYSTEM) + + def run(self): + return Agent(instructions=SYSTEM) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, "class", agents["Bot.direct"].Fields["instructions"].Value, "the class body sees its own attribute") + assert.Equal(t, "module", agents["Bot.run.Agent"].Fields["instructions"].Value, "a method sees the module name, not the class attribute") +} + +func TestPythonReassignedConstantsAreReadOnlyWithDistinctIDs(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent +NAME = "one" +NAME = "two" +support = Agent(instructions=NAME) +`)}}) + require.NoError(t, err) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + require.Contains(t, constants, "NAME") + require.Contains(t, constants, "NAME#2", "each assignment keeps an id of its own") + for id, constant := range constants { + assert.Equal(t, adapter.CapabilityReadOnly, constant.Capability, id) + assert.Equal(t, "reassigned constants are read-only", constant.Reason, id) + } + assert.NotEqual(t, adapter.CapabilityDirect, root.Fields["agents"].Items[0].Fields["instructions"].Capability) +} + +func TestPythonStandaloneFastMCPImportIsFollowed(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from fastmcp import FastMCP +mcp = FastMCP("server") + +@mcp.tool() +def run(value: str): + """Run.""" + return value + +@mcp.prompt() +def greet(name: str): + """Greet.""" + return name +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["tools"].Items, 1, "the standalone fastmcp package is a FastMCP server") + assert.Equal(t, "run", root.Fields["tools"].Items[0].Fields["id"].Value) + require.Len(t, root.Fields["prompts"].Items, 1) + assert.Equal(t, "greet", root.Fields["prompts"].Items[0].Fields["id"].Value) +} + +func TestPythonTuplesAreWrittenAsTuples(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +PAIR = ("a", "b") +ONE = ("a",) +LIST = ["a"] +`)}}) + require.NoError(t, err) + encode := func(id string, value []any) string { + node, err := root.Resolve(`constants[id="` + id + `"]`) + require.NoError(t, err) + encoded, err := node.Encode(value) + require.NoError(t, err) + return string(encoded) + } + assert.Equal(t, `("x", "y")`, encode("PAIR", []any{"x", "y"}), "a tuple stays a tuple") + assert.Equal(t, `("z",)`, encode("ONE", []any{"z"}), "a tuple of one item keeps its comma") + assert.Equal(t, `["z"]`, encode("LIST", []any{"z"})) +} + +func TestPythonFrameworkStarImportBindsItsSymbols(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import * +support = Agent(instructions="Help") +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1, "a star import of a framework binds its symbols") + assert.Equal(t, "Help", root.Fields["agents"].Items[0].Fields["instructions"].Value) +} + +func TestPythonClassAttributeClientsResolveThroughSelf(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from openai import OpenAI + +class Assistant: + client = OpenAI() + + def ask(self): + return self.client.chat.completions.create(model="gpt", messages=[{"role": "user", "content": "one"}]) +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["prompts"].Items, 1, "self.client names the class attribute client") + assert.Equal(t, "Assistant.ask.self.client.chat.completions.create", root.Fields["prompts"].Items[0].Fields["id"].Value) +} + +func TestPythonDocstringAfterACommentIsTheDescription(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from langchain_core.tools import tool + +@tool +def search(query: str): + # The docstring follows this comment. + """Search the docs.""" + return query +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["tools"].Items, 1) + assert.Equal(t, "Search the docs.", root.Fields["tools"].Items[0].Fields["description"].Value) +} + +func TestPythonImportStaysVisibleBeforeALaterShadowingAssignment(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +from .values import TEXT +first = Agent(instructions=TEXT) +TEXT = "Local" +second = Agent(instructions=TEXT) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("TEXT = \"Imported\"\n")}, + }, + }) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, "Imported", agents["first"].Fields["instructions"].Value, "a use before the assignment sees the import") + assert.Equal(t, adapter.CapabilityDirect, agents["first"].Fields["instructions"].Capability) + assert.Equal(t, "Local", agents["second"].Fields["instructions"].Value, "a use after the assignment sees the local value") +} + +func TestPythonPositionalArgumentsFollowTheFrameworkSignatures(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent +from strands import Agent as StrandsAgent +support = Agent("support", "You are helpful.") +strands = StrandsAgent("claude-model", system_prompt="Follow the policy.") +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, "support", agents["support"].Fields["name"].Value) + assert.Equal(t, "You are helpful.", agents["support"].Fields["instructions"].Value, "the Agents SDK takes instructions second") + assert.Equal(t, "claude-model", agents["strands"].Fields["model"].Value, "Strands takes the model first") + assert.Equal(t, "Follow the policy.", agents["strands"].Fields["system_prompt"].Value, "a positional model does not replace the keyword prompt") +} + +func TestPythonFrameworkAliasesFollowTheirImportPosition(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from crewai import Agent +crew = Agent(role="Writer", goal="Write", backstory="Years of {topic}") + +def build(): + from pydantic_ai import Agent + return Agent("gpt-5") + +from agents import Agent +support = Agent(name="support", instructions="Help") +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + require.Len(t, agents, 3) + assert.Equal(t, "Years of {{topic}}", agents["crew"].Fields["backstory"].Value, "the first import is the CrewAI agent at that point") + assert.Equal(t, "gpt-5", agents["build.Agent"].Fields["model"].Value, "a local import applies inside its function") + assert.Equal(t, "Help", agents["support"].Fields["instructions"].Value, "the later import is the Agents SDK agent") + assert.Nil(t, agents["support"].Fields["backstory"]) +} + +func TestPythonDottedMutationsFreezeTheRootConstant(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +SETTINGS = {"values": {"a": 1}} +SETTINGS["values"].update({"b": 2}) + +class Bot: + items = ["a"] + + def add(self): + self.items.append("b") +`)}}) + require.NoError(t, err) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityReadOnly, constants["SETTINGS"].Capability, "a mutating call through a subscript freezes the constant") + assert.Equal(t, adapter.CapabilityReadOnly, constants["Bot.items"].Capability, "a mutating call through self freezes the class attribute") +} + +func TestPythonSelfAttributeWritesFreezeTheClassAttribute(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +class Bot: + items = ["a"] + names = ["n"] + greeting = "Hi" + kept = "same" + + def add(self): + self.items[0] = "b" + + def more(self): + self.names += ["m"] + + def reset(self): + self.greeting = "Hello" +`)}}) + require.NoError(t, err) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityReadOnly, constants["Bot.items"].Capability, "a subscript write through self freezes the class attribute") + assert.Equal(t, adapter.CapabilityReadOnly, constants["Bot.names"].Capability, "an augmented assignment through self freezes the class attribute") + assert.Equal(t, adapter.CapabilityReadOnly, constants["Bot.greeting"].Capability, "an assignment through self freezes the class attribute") + assert.Equal(t, adapter.CapabilityDirect, constants["Bot.kept"].Capability, "an attribute that no method writes stays direct") + assert.Nil(t, constants["self"], "self is not a constant") +} + +func TestPythonClassBodiesSeeOnlyEarlierImports(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from crewai import Agent + +class Team: + writer = Agent(role="Writer", goal="Write", backstory="Years of {topic}") + + def run(self): + return Agent(name="runner", instructions="Run") + +from agents import Agent +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, "Years of {{topic}}", agents["Team.writer"].Fields["backstory"].Value, "a class body runs where it is and sees the CrewAI import") + assert.Equal(t, "Run", agents["runner"].Fields["instructions"].Value, "a method runs later and sees the Agents SDK import") +} + +func TestPythonParenthesizedValuesIgnoreComments(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +PROMPT = ( # the greeting + "Help the customer." +) +`)}}) + require.NoError(t, err) + prompt, err := root.Resolve(`constants[id="PROMPT"]`) + require.NoError(t, err) + assert.Equal(t, "Help the customer.", prompt.Value) + assert.Equal(t, adapter.CapabilityDirect, prompt.Capability) +} + +func TestPythonAwaitedCallsKeepTheirAssignmentName(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from openai import OpenAI +client = OpenAI() + +async def ask(): + reply = await client.chat.completions.create(model="gpt", messages=[{"role": "user", "content": "Hi"}]) + other = (client.chat.completions.create(model="gpt", messages=[{"role": "user", "content": "Yo"}])) + return reply, other +`)}}) + require.NoError(t, err) + ids := []string{} + for _, prompt := range root.Fields["prompts"].Items { + ids = append(ids, prompt.Fields["id"].Value.(string)) + } + assert.ElementsMatch(t, []string{"ask.reply", "ask.other"}, ids, "await and parentheses keep the variable name") +} + +func TestPythonPositionalMessagesAreAMessageList(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from strands import Agent +agent = Agent("claude-model", [{"role": "user", "content": "Hi"}], system_prompt="Follow the policy.") +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1) + messages := root.Fields["agents"].Items[0].Fields["messages"] + assert.Equal(t, []syncdomain.Message{{Role: syncdomain.RoleUser, Content: "Hi"}}, messages.Value, "a positional messages list is read as the keyword one") +} + +func TestPythonImportedValuesFollowTheScopeOfTheirImport(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +from .values import TEXT + +def build(): + from .other import TEXT + return Agent(instructions=TEXT) + +support = Agent(instructions=TEXT) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("TEXT = \"Outer\"\n")}, + "src/other.py": {Path: "src/other.py", Data: []byte("TEXT = \"Inner\"\n")}, + }, + }) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, "Inner", agents["build.Agent"].Fields["instructions"].Value, "a local import binds its name inside the function") + assert.Equal(t, "Outer", agents["support"].Fields["instructions"].Value, "module-level code keeps the module-level import") +} + +func TestPythonImportsStayVisibleBeforeAShadowingAssignmentInAnyScope(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +from .values import TEXT + +class Team: + first = Agent(instructions=TEXT) + TEXT = "class" + second = Agent(instructions=TEXT) + +def build(): + from .values import TEXT + a = Agent(instructions=TEXT) + TEXT = "local" + b = Agent(instructions=TEXT) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("TEXT = \"Imported\"\n")}, + }, + }) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, "Imported", agents["Team.first"].Fields["instructions"].Value, "a class body sees the module import until it binds the name") + assert.Equal(t, "class", agents["Team.second"].Fields["instructions"].Value) + assert.Equal(t, "Imported", agents["build.a"].Fields["instructions"].Value, "a local import is visible before the local assignment") + assert.Equal(t, "local", agents["build.b"].Fields["instructions"].Value) +} + +func TestPythonMutatedLocalImportIsReadOnly(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent + +def build(): + from .values import TEXT + TEXT += "!" + return Agent(instructions=TEXT) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("TEXT = \"Help\"\n")}, + }, + }) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1) + instructions := root.Fields["agents"].Items[0].Fields["instructions"] + assert.Equal(t, adapter.CapabilityReadOnly, instructions.Capability, "a mutation of a function-local import freezes it") + assert.Equal(t, "mutated constants are read-only", instructions.Reason) +} + +func TestPythonReassignedAndMutatedConstantsStayPositional(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +NAME = "one" +NAME = "two" +NAME += "!" +`)}}) + require.NoError(t, err) + first, err := root.Resolve(`constants[id="NAME"]`) + require.NoError(t, err) + assert.Equal(t, adapter.CapabilityReadOnly, first.Capability) + assert.True(t, adapter.IsPositionalID(first), "the read-only mark does not replace the positional id") +} + +func TestPythonParenthesizedMessagesAndConstructors(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from openai import OpenAI +client = (OpenAI()) +reply = client.chat.completions.create(model="gpt", messages=([{"role": "user", "content": "Hi"}])) +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["prompts"].Items, 1, "a parenthesized constructor is an instance") + messages := root.Fields["prompts"].Items[0].Fields["messages"] + assert.Equal(t, []syncdomain.Message{{Role: syncdomain.RoleUser, Content: "Hi"}}, messages.Value, "a parenthesized messages list unwraps") +} + +func TestPythonChainedAssignmentsBindEveryNameAndCollectOnce(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent +from openai import OpenAI +first = second = Agent(instructions="Help") +primary = backup = OpenAI() +one = primary.chat.completions.create(model="gpt", messages=[{"role": "user", "content": "A"}]) +two = backup.chat.completions.create(model="gpt", messages=[{"role": "user", "content": "B"}]) +`)}}) + require.NoError(t, err) + agents := []string{} + for _, agent := range root.Fields["agents"].Items { + agents = append(agents, agent.Fields["id"].Value.(string)) + } + assert.Equal(t, []string{"first"}, agents, "a chained constructor is one resource, named by the outer name") + prompts := []string{} + for _, prompt := range root.Fields["prompts"].Items { + prompts = append(prompts, prompt.Fields["id"].Value.(string)) + } + assert.ElementsMatch(t, []string{"one", "two"}, prompts, "every name of a chained client binding is a client") +} + +func TestPythonWithAsBindsAClient(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from openai import OpenAI + +def ask(): + with OpenAI() as client: + reply = client.chat.completions.create(model="gpt", messages=[{"role": "user", "content": "Hi"}]) + return reply +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["prompts"].Items, 1, "a with-as binding is a client") + assert.Equal(t, "ask.reply", root.Fields["prompts"].Items[0].Fields["id"].Value) +} + +func TestPythonFunctionsSeeLaterModuleConstants(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +def build(): + return Agent(instructions=SYSTEM) + +early = Agent(instructions=SYSTEM) +SYSTEM = "Help" +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, "Help", agents["build.Agent"].Fields["instructions"].Value, "a function body runs after the module loads") + assert.NotEqual(t, adapter.CapabilityDirect, agents["early"].Fields["instructions"].Capability, "module-level code before the constant does not see it") +} + +func TestPythonCommonListMutatorsFreezeConstants(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +ITEMS = ["b", "a"] +ITEMS.sort() +NAMES = ["x"] +NAMES.remove("x") +KEPT = ["k"] +`)}}) + require.NoError(t, err) + capability := func(id string) adapter.Capability { + node, err := root.Resolve(`constants[id="` + id + `"]`) + require.NoError(t, err) + return node.Capability + } + assert.Equal(t, adapter.CapabilityReadOnly, capability("ITEMS"), "sort mutates in place") + assert.Equal(t, adapter.CapabilityReadOnly, capability("NAMES"), "remove mutates in place") + assert.Equal(t, adapter.CapabilityDirect, capability("KEPT")) +} + +func TestPythonParametersHideModuleConstants(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent +SYSTEM = "Module" + +def build(SYSTEM, other="x"): + return Agent(instructions=SYSTEM) + +def plain(): + return Agent(instructions=SYSTEM) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.NotEqual(t, adapter.CapabilityDirect, agents["build.Agent"].Fields["instructions"].Capability, "a parameter hides the module constant") + assert.Equal(t, "Module", agents["plain.Agent"].Fields["instructions"].Value) +} + +func TestPythonChainedPromptOrdinalsArePositional(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from langchain_core.prompts import PromptTemplate +prompts = PromptTemplate.from_template("one") | PromptTemplate.from_template("two") +`)}}) + require.NoError(t, err) + first, err := root.Resolve(`prompts[id="prompts#1"]`) + require.NoError(t, err) + assert.True(t, adapter.IsPositionalID(first), "an ordinal in a chain is a position") +} + +func TestPythonParametersHideNamesBeforeTheBodyReadsThem(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent +TEXT = "Module" + +def build(TEXT): + local = TEXT + return Agent(instructions=local) + +handler = lambda TEXT: Agent(instructions=TEXT) +support = Agent(instructions=TEXT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.NotEqual(t, adapter.CapabilityDirect, agents["build.Agent"].Fields["instructions"].Capability, "a local alias of a parameter is dynamic") + assert.NotEqual(t, adapter.CapabilityDirect, agents["Agent"].Fields["instructions"].Capability, "a lambda parameter is dynamic") + assert.Equal(t, "Module", agents["support"].Fields["instructions"].Value, "a lambda parameter does not hide the module constant outside the lambda") +} + +func TestPythonLambdaParametersBindOnlyInsideTheLambda(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" + +def build(): + pick = lambda TEXT: TEXT + inner = lambda TEXT: Agent(instructions=TEXT) + outer = Agent(instructions=TEXT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + require.Len(t, agents, 2) + assert.Equal(t, adapter.CapabilityReadOnly, agents["build.Agent"].Fields["instructions"].Capability, "the lambda body reads its parameter") + assert.Equal(t, "module", agents["build.outer"].Fields["instructions"].Value, "a lambda parameter does not hide the constant in the enclosing function") +} + +func TestPythonExceptAliasesHideTheOuterConstant(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +error = "module" + +def handle(): + try: + pass + except (ValueError, KeyError) as error: + inside = Agent(instructions=error) + after = Agent(instructions=error) + +try: + pass +except Exception as error: + pass +module_level = Agent(instructions=error) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + require.Len(t, agents, 3) + assert.Equal(t, adapter.CapabilityReadOnly, agents["handle.inside"].Fields["instructions"].Capability) + assert.Equal(t, adapter.CapabilityReadOnly, agents["handle.after"].Fields["instructions"].Capability, "the alias binds the name in the function") + assert.Equal(t, adapter.CapabilityReadOnly, agents["module_level"].Fields["instructions"].Capability, "the alias rebinds the module constant") +} + +func TestPythonMutationThroughAnAliasFreezesEveryName(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from .values import ITEMS +from .values import ITEMS as ALSO +from .values import TEXT +ITEMS.append("b") +WRAPPED = [ALSO] +LOCAL = ["a"] +COPY = LOCAL +COPY.append("b") +PLAIN = [TEXT] +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("ITEMS = [\"a\"]\nTEXT = \"Hello\"\n")}, + }, + }) + require.NoError(t, err) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityReadOnly, constants["WRAPPED"].Capability, "the second import of a mutated value is read-only") + assert.Equal(t, adapter.CapabilityReadOnly, constants["LOCAL"].Capability, "a mutation through a local alias freezes the origin") + assert.Equal(t, adapter.CapabilityDirect, constants["PLAIN"].Capability, "an untouched import stays writable") +} + +func TestPythonDeleteStatementsFreezeTheCollection(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +ITEMS = ["a", "b"] +CONFIG = {"key": "value", "other": "kept"} +PAIR = ["x", "y"] +NAME = "gone" +KEPT = ["z"] +del ITEMS[0] +del CONFIG["key"], PAIR[1] +del NAME +`)}}) + require.NoError(t, err) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + for _, id := range []string{"ITEMS", "CONFIG", "PAIR", "NAME"} { + assert.Equal(t, adapter.CapabilityReadOnly, constants[id].Capability, id) + } + assert.Equal(t, adapter.CapabilityDirect, constants["KEPT"].Capability) +} + +func TestPythonGlobalAndNonlocalAssignmentsRebindTheOuterConstant(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" +KEPT = "kept" + +def rebind(): + global TEXT + TEXT = "changed" + +def outer(): + INNER = "outer" + def inner(): + nonlocal INNER + INNER = "inner" + return Agent(instructions=INNER) + return Agent(instructions=INNER) + +def local(): + KEPT = "local" + return Agent(instructions=KEPT) + +module_level = Agent(instructions=TEXT) +kept = Agent(instructions=KEPT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityReadOnly, agents["module_level"].Fields["instructions"].Capability, "a global assignment rebinds the module constant") + assert.Equal(t, adapter.CapabilityReadOnly, constants["TEXT"].Capability) + assert.Equal(t, "reassigned constants are read-only", constants["TEXT"].Reason) + assert.Nil(t, constants["rebind.TEXT"], "the global assignment is not a local of the function") + assert.Equal(t, adapter.CapabilityReadOnly, agents["outer.Agent"].Fields["instructions"].Capability, "a nonlocal assignment rebinds the constant of the enclosing function") + assert.Equal(t, adapter.CapabilityReadOnly, agents["outer.inner.Agent"].Fields["instructions"].Capability) + assert.Equal(t, "kept", agents["kept"].Fields["instructions"].Value, "a plain local assignment leaves the module constant") + assert.Equal(t, "local", agents["local.Agent"].Fields["instructions"].Value) +} + +func TestPythonWalrusPromptsKeepTheirOwnName(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from openai import OpenAI + +client = OpenAI() +batch = [reply := client.chat.completions.create(model="gpt-5", messages=[{"role": "user", "content": "Hi"}])] +chain = alias = client.chat.completions.create(model="gpt-5", messages=[{"role": "user", "content": "Chain"}]) +`)}}) + require.NoError(t, err) + ids := []string{} + for _, prompt := range root.Fields["prompts"].Items { + ids = append(ids, prompt.Fields["id"].Value.(string)) + } + assert.ElementsMatch(t, []string{"reply", "chain"}, ids, "a walrus inside a list names its prompt, and a chain keeps the outer name") +} + +func TestPythonMessagesSkipInnerLocalBindings(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from openai import OpenAI + +client = OpenAI() +MESSAGES = [{"role": "user", "content": "module"}] +handlers = [lambda MESSAGES: client.chat.completions.create(model="gpt-5", messages=MESSAGES)] +replies = [client.chat.completions.create(model="gpt-5", messages=MESSAGES) for MESSAGES in batches] +plain = client.chat.completions.create(model="gpt-5", messages=MESSAGES) +`)}}) + require.NoError(t, err) + prompts := map[string]*adapter.Node{} + for _, prompt := range root.Fields["prompts"].Items { + prompts[prompt.Fields["id"].Value.(string)] = prompt + } + require.Len(t, prompts, 3) + // A request inside a lambda or a comprehension has no variable name, so + // the callable names it. + assert.Equal(t, adapter.CapabilityReadOnly, prompts["client.chat.completions.create"].Fields["messages"].Capability, "a lambda parameter is not the module messages") + assert.Equal(t, adapter.CapabilityReadOnly, prompts["client.chat.completions.create#2"].Fields["messages"].Capability, "a comprehension target is not the module messages") + assert.Equal(t, []syncdomain.Message{{Role: syncdomain.RoleUser, Content: "module"}}, prompts["plain"].Fields["messages"].Value) +} + +func TestPythonLambdaDefaultsAndFirstIterablesSeeTheOuterName(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" +ITEMS = ["a"] +defaulted = lambda TEXT=Agent(instructions=TEXT): TEXT +body = lambda TEXT: Agent(instructions=TEXT) +first = [Agent(instructions=TEXT) for TEXT in Agent(instructions=TEXT).items] +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + require.Len(t, agents, 4) + assert.Equal(t, "module", agents["Agent"].Fields["instructions"].Value, "a lambda default runs in the enclosing scope") + assert.Equal(t, adapter.CapabilityReadOnly, agents["Agent#2"].Fields["instructions"].Capability, "the lambda body reads its parameter") + assert.Equal(t, adapter.CapabilityReadOnly, agents["Agent#3"].Fields["instructions"].Capability, "the comprehension body reads its target") + assert.Equal(t, "module", agents["Agent#4"].Fields["instructions"].Value, "the first iterable runs in the enclosing scope") +} + +func TestPythonImportsOfReassignedAndMutatedNamesAreReadOnlyBindings(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +from .values import TEXT, ITEMS, PLAIN + +support = Agent(instructions=TEXT) +WRAPPED = [ITEMS] +other = Agent(instructions=PLAIN) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("TEXT = \"first\"\nTEXT = \"second\"\nITEMS = [\"a\"]\nITEMS.append(\"b\")\nPLAIN = \"plain\"\n")}, + }, + }) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + instructions := agents["support"].Fields["instructions"] + assert.Equal(t, "first", instructions.Value, "the import binds the first literal") + assert.Equal(t, adapter.CapabilityReadOnly, instructions.Capability) + assert.Equal(t, "reassigned constants are read-only", instructions.Reason) + assert.Empty(t, instructions.Code, "a reassigned name is not an unresolved import") + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityReadOnly, constants["WRAPPED"].Capability, "a mutated import stays a read-only binding") + assert.Equal(t, "plain", agents["other"].Fields["instructions"].Value) +} + +func TestPythonNonlocalBindsTheNearestFunctionThatHasTheName(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +def outer(): + TEXT = "outer" + def middle(): + def inner(): + nonlocal TEXT + TEXT = "inner" + return Agent(instructions=TEXT) + return Agent(instructions=TEXT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + require.Len(t, agents, 2) + assert.Equal(t, adapter.CapabilityReadOnly, agents["outer.Agent"].Fields["instructions"].Capability, "the nonlocal skips the function without the name and rebinds the outer constant") + assert.Equal(t, adapter.CapabilityReadOnly, agents["outer.middle.Agent"].Fields["instructions"].Capability) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Nil(t, constants["outer.middle.TEXT"], "the function without the name gets no binding") + assert.NotNil(t, constants["outer.TEXT#2"], "the outer constant is reassigned") +} + +func TestPythonDefsAndClassesBindTheirNames(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" +NAME = "module" + +def TEXT(): + pass + +class NAME: + pass + +rebound = Agent(instructions=TEXT) +classed = Agent(instructions=NAME) + +def outer(): + def LABEL(): + pass + def middle(): + def inner(): + nonlocal LABEL + LABEL = "inner" + return Agent(instructions=LABEL) + return middle +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + require.Len(t, agents, 3) + assert.Equal(t, adapter.CapabilityReadOnly, agents["rebound"].Fields["instructions"].Capability, "a def rebinds the constant") + assert.Equal(t, adapter.CapabilityReadOnly, agents["classed"].Fields["instructions"].Capability, "a class rebinds the constant") + assert.Equal(t, adapter.CapabilityReadOnly, agents["outer.middle.Agent"].Fields["instructions"].Capability, "the nonlocal binds in the function whose def owns the name") + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityReadOnly, constants["TEXT"].Capability) + assert.Nil(t, constants["outer.middle.LABEL"], "the function between gets no binding") +} + +func TestPythonStarImportDoesNotSupplyFunctionLocals(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +from .values import * + +def build(TEXT, items): + LOCAL = items[0] + first = Agent(instructions=TEXT) + second = Agent(instructions=LOCAL) + third = Agent(instructions=OTHER) +`)}, + Dependencies: map[string]adapter.Document{"src/values.py": {Path: "src/values.py", Data: []byte("OTHER = \"Help\"\n")}}, + }) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + require.Len(t, agents, 3) + for _, id := range []string{"build.first", "build.second"} { + instructions := agents[id].Fields["instructions"] + assert.Equal(t, adapter.CapabilityReadOnly, instructions.Capability, id) + assert.Equal(t, "dynamic expression has no safe inverse", instructions.Reason, "a local is dynamic, not a name of the star import") + assert.Empty(t, instructions.Code, id) + } + assert.Contains(t, agents["build.third"].Fields["instructions"].Reason, "star import cannot resolve OTHER statically") +} + +func TestPythonNonlocalBindsAnImportedEnclosingName(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" + +def outer(): + from .values import TEXT + def inner(): + nonlocal TEXT + TEXT = "inner" + return Agent(instructions=TEXT) + +plain = Agent(instructions=TEXT) +`)}, + Dependencies: map[string]adapter.Document{"src/values.py": {Path: "src/values.py", Data: []byte("TEXT = \"imported\"\n")}}, + }) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, adapter.CapabilityReadOnly, agents["outer.Agent"].Fields["instructions"].Capability, "the nonlocal rebinds the imported name of the enclosing function") + assert.Equal(t, "module", agents["plain"].Fields["instructions"].Value, "the module constant stays") + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityDirect, constants["TEXT"].Capability, "the module constant is not reassigned") +} + +func TestPythonGlobalStatementsScopeReadsAndMutations(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +ITEMS = ["module"] +TEXT = "module" + +def outer(): + ITEMS = ["outer"] + TEXT = "outer" + def inner(): + global ITEMS, TEXT + ITEMS.append("x") + return Agent(instructions=TEXT) + return Agent(instructions=TEXT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, "module", agents["outer.inner.Agent"].Fields["instructions"].Value, "a global read skips the local of the enclosing function") + assert.Equal(t, "outer", agents["outer.Agent"].Fields["instructions"].Value) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityReadOnly, constants["ITEMS"].Capability, "a global mutation freezes the module constant") + assert.Equal(t, adapter.CapabilityDirect, constants["outer.ITEMS"].Capability, "the local of the enclosing function is untouched") +} + +func TestPythonImportedAliasesShareMutations(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from .values import ITEMS, ALSO, OTHER +ITEMS.append("b") +WRAPPED = [ALSO] +PLAIN = [OTHER] +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("ITEMS = [\"a\"]\nALSO = ITEMS\nOTHER = [\"c\"]\n")}, + }, + }) + require.NoError(t, err) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityReadOnly, constants["WRAPPED"].Capability, "an alias of the mutated import binds the same object") + assert.Equal(t, adapter.CapabilityDirect, constants["PLAIN"].Capability) +} + +func TestPythonNonlocalSkipsAGlobalOfAFunctionBetween(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" + +def outer(): + TEXT = "outer" + def middle(): + global TEXT + def inner(): + nonlocal TEXT + TEXT = "inner" + return inner + return Agent(instructions=TEXT) + +plain = Agent(instructions=TEXT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, adapter.CapabilityReadOnly, agents["outer.Agent"].Fields["instructions"].Capability, "the nonlocal rebinds the local of outer, past the global of middle") + assert.Equal(t, "module", agents["plain"].Fields["instructions"].Value, "the global of middle does not move the nonlocal write to the module") +} + +func TestPythonAugmentedAssignmentsBindInTheirFunction(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" + +def outer(): + def middle(TEXT): + TEXT += " more" + def inner(): + nonlocal TEXT + TEXT = "inner" + return inner + return Agent(instructions=TEXT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, "module", agents["outer.Agent"].Fields["instructions"].Value, "the nonlocal lands on middle, which binds the name, so outer reads the module constant") + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityDirect, constants["TEXT"].Capability, "the module constant is not rebound") +} + +func TestPythonImportOfAnAliasOfAMutatedNameIsReadOnly(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{ + Primary: adapter.Document{Path: "src/main.py", Data: []byte(` +from agents import Agent +from .values import ALSO, TEXT_ALSO + +WRAPPED = [ALSO] +support = Agent(instructions=TEXT_ALSO) +`)}, + Dependencies: map[string]adapter.Document{ + "src/values.py": {Path: "src/values.py", Data: []byte("ITEMS = [\"a\"]\nALSO = ITEMS\nITEMS.append(\"b\")\nTEXT = \"first\"\nTEXT_ALSO = TEXT\nTEXT = \"second\"\n")}, + }, + }) + require.NoError(t, err) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityReadOnly, constants["WRAPPED"].Capability, "the alias of a mutated name imports the read-only origin") + instructions := root.Fields["agents"].Items[0].Fields["instructions"] + assert.Equal(t, "first", instructions.Value, "the alias of a reassigned name imports its first literal") + assert.Equal(t, adapter.CapabilityReadOnly, instructions.Capability) + assert.Empty(t, instructions.Code, "the alias is not an unresolved import") +} + +func TestPythonBareAnnotationsBindTheirName(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" +OTHER = "module" + +def build(): + TEXT: str + return Agent(instructions=TEXT) + +def assigned(): + OTHER: str + OTHER = "local" + return Agent(instructions=OTHER) + +class Bot: + TEXT: str + annotated = Agent(instructions=TEXT) + +TEXT: str +plain = Agent(instructions=TEXT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, adapter.CapabilityReadOnly, agents["build.Agent"].Fields["instructions"].Capability, "a bare annotation makes the name local") + assert.Equal(t, "local", agents["assigned.Agent"].Fields["instructions"].Value, "an annotation with a later assignment is one binding") + assert.Equal(t, "module", agents["Bot.annotated"].Fields["instructions"].Value, "a bare annotation in a class body binds nothing") + assert.Equal(t, "module", agents["plain"].Fields["instructions"].Value, "a bare annotation at module level binds nothing") + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityDirect, constants["assigned.OTHER"].Capability, "the annotation does not count as a reassignment") +} + +func TestPythonExceptGroupAliasesBindTheirName(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +error = "module" + +def handle(): + try: + pass + except* ExceptionGroup as error: + pass + return Agent(instructions=error) +`)}}) + require.NoError(t, err) + require.Len(t, root.Fields["agents"].Items, 1) + assert.Equal(t, adapter.CapabilityReadOnly, root.Fields["agents"].Items[0].Fields["instructions"].Capability, "an except* alias binds the name in the function") +} + +func TestPythonAugmentedAndDeleteWritesInAFunctionBindLocals(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" +ITEMS = ["a"] +COUNT = ["b"] + +def build(): + TEXT += " more" + del ITEMS + return Agent(instructions=TEXT) + +def shared(): + global COUNT + del COUNT + +plain = Agent(instructions=TEXT) +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, adapter.CapabilityReadOnly, agents["build.Agent"].Fields["instructions"].Capability, "the augmented write makes the name a local") + assert.Equal(t, "module", agents["plain"].Fields["instructions"].Value) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityDirect, constants["TEXT"].Capability, "a local augmented write does not freeze the module constant") + assert.Equal(t, adapter.CapabilityDirect, constants["ITEMS"].Capability, "a local delete does not freeze the module constant") + assert.Equal(t, adapter.CapabilityReadOnly, constants["COUNT"].Capability, "a delete under a global statement mutates the module constant") +} + +func TestPythonClassBodiesSeeTheModuleConstantBeforeTheClassBindsIt(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +TEXT = "module" + +class Bot: + early = Agent(instructions=TEXT) + TEXT = "class" + late = Agent(instructions=TEXT) + +class Twice: + early = Agent(instructions=TEXT) + TEXT = "one" + TEXT = "two" + late = Agent(instructions=TEXT) + +class Dynamic: + early = Agent(instructions=TEXT) + TEXT = compute() +`)}}) + require.NoError(t, err) + agents := map[string]*adapter.Node{} + for _, agent := range root.Fields["agents"].Items { + agents[agent.Fields["id"].Value.(string)] = agent + } + assert.Equal(t, "module", agents["Bot.early"].Fields["instructions"].Value, "the class body sees the module constant until it binds the name") + assert.Equal(t, "class", agents["Bot.late"].Fields["instructions"].Value) + assert.Equal(t, "module", agents["Twice.early"].Fields["instructions"].Value, "a reassigned class attribute still leaves the earlier use to the module") + assert.Equal(t, adapter.CapabilityReadOnly, agents["Twice.late"].Fields["instructions"].Capability) + assert.Equal(t, "module", agents["Dynamic.early"].Fields["instructions"].Value, "a dynamic class attribute still leaves the earlier use to the module") +} + +func TestPythonReassignedAliasesDoNotSpreadMutations(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +ITEMS = ["a"] +ALSO = ITEMS +ALSO = ["b"] +ALSO.append("c") +WRAPPED = [ITEMS] +`)}}) + require.NoError(t, err) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, adapter.CapabilityDirect, constants["ITEMS"].Capability, "a name that is assigned again is no alias of its first value") + assert.Equal(t, adapter.CapabilityDirect, constants["WRAPPED"].Capability) +} + +func TestPythonWalrusValuesReadAsConstants(t *testing.T) { + root, err := parsePythonBundle(adapter.SourceBundle{Primary: adapter.Document{Path: "main.py", Data: []byte(` +from agents import Agent + +NAME = (OTHER := "text") +ITEMS = [first := "hello", "world"] +support = Agent(instructions=NAME) +`)}}) + require.NoError(t, err) + constants := map[string]*adapter.Node{} + for _, constant := range root.Fields["constants"].Items { + constants[constant.Fields["id"].Value.(string)] = constant + } + assert.Equal(t, "text", constants["NAME"].Value, "the value of a walrus is the value of the assignment") + assert.Equal(t, adapter.CapabilityDirect, constants["NAME"].Capability) + assert.Equal(t, "text", constants["OTHER"].Value) + assert.Equal(t, adapter.CapabilityDirect, constants["ITEMS"].Capability, "a walrus inside a list is a literal item") + assert.Equal(t, []any{"hello", "world"}, constants["ITEMS"].Value) + assert.Equal(t, "text", root.Fields["agents"].Items[0].Fields["instructions"].Value) +} diff --git a/internal/sync/adapter/resolve.go b/internal/sync/adapter/resolve.go index 0b769cf82..a7906ecb2 100644 --- a/internal/sync/adapter/resolve.go +++ b/internal/sync/adapter/resolve.go @@ -22,6 +22,10 @@ type indexedResolution struct { path string collectionLength int item *Node + // identity replaces the identity of the item when it is set. A positional + // id uses the count of the resources that share its name, so a resource + // added beside it changes the selection and an edit to it does not. + identity any } // resolve returns the node at a path and the index steps that it took. scope @@ -80,6 +84,7 @@ func (root *Node) resolve(text, scope string) (*Node, []indexedResolution, error if len(step.filters) == 0 { continue } + collection := current var matches []*Node for _, item := range current.Items { matched := true @@ -116,6 +121,17 @@ func (root *Node) resolve(text, scope string) (*Node, []indexedResolution, error ) case 1: current = matches[0] + // A positional id, such as Agent or Agent#2, is the position of a + // resource that has no name of its own, so the selection is as + // fragile as an index and is recorded like one. + if IsPositionalID(current) { + indexes = append(indexes, indexedResolution{ + path: fmt.Sprintf("%s#%d", scope, len(indexes)), + collectionLength: len(collection.Items), + item: current, + identity: positionalIdentity(collection, current), + }) + } default: return nil, nil, syncdomain.NewProblem( syncdomain.ErrorPathAmbiguous, @@ -133,7 +149,10 @@ func indexedSelections(indexes []indexedResolution, selectedRoot *Node) ([]syncd selections := make([]syncdomain.IndexedSelection, 0, len(indexes)) var ancestors []string for _, index := range indexes { - identity := directSelectionIdentity(index.item) + identity := index.identity + if identity == nil { + identity = directSelectionIdentity(index.item) + } if identity == nil { // Without scalar identity, bind the nearest selected resource root and prior indexed ancestors. identity = struct { @@ -158,6 +177,28 @@ func indexedSelections(indexes []indexedResolution, selectedRoot *Node) ([]syncd return selections, nil } +// positionalIdentity identifies a resource with a positional id by the name +// that it fell back to and the count of the resources in its collection that +// fell back to the same name. +func positionalIdentity(collection, item *Node) any { + id, _ := item.Fields["id"].Value.(string) + base, _, _ := strings.Cut(id, "#") + siblings := 0 + for _, candidate := range collection.Items { + if !IsPositionalID(candidate) { + continue + } + if text, _ := candidate.Fields["id"].Value.(string); strings.HasPrefix(text, base) && + (len(text) == len(base) || text[len(base)] == '#') { + siblings++ + } + } + return struct { + Positional string `json:"positional"` + Siblings int `json:"siblings"` + }{Positional: base, Siblings: siblings} +} + func directSelectionIdentity(node *Node) any { if node == nil || node.Fields == nil { return nil @@ -168,6 +209,10 @@ func directSelectionIdentity(node *Node) any { if !ok || !scalarValue(field.Value) { continue } + // A positional id is a position, not an identity. + if key == "id" && IsPositionalID(node) { + continue + } identity[key] = field.Value } if len(identity) == 0 {