From fbdacd00f6cefbe954047ec6e8874ec19fd3e710 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 5 Nov 2026 11:13:06 -0500 Subject: [PATCH 01/15] feat(go): model the L3/L4 dataflow substrate GoType.kind rejected "alias", so real -a 1 output from any project declaring a type alias failed to parse. Widen it to all four kinds the analyzer emits. Add the dataflow fields that -a 3/-a 4 output carried but the models dropped: body-node id/of/parent/is_deferred, per-callable cfg/cdg/ddg/summary, application-scope param_in/param_out/external_symbols, and envelope-scope degraded. The overlays are Optional so "not computed" stays distinct from "computed, no edges". Verified lossless against cango 0.3.0 -a 4 on testdata/dataflow: 319 callables, 1299 cfg / 666 cdg / 2854 ddg / 704 summary edges and 3324 body nodes all round-trip, with no unmodelled wire fields. Refs #436 --- cldk/models/go/__init__.py | 29 +++++- cldk/models/go/models.py | 176 ++++++++++++++++++++++++++++++++++--- 2 files changed, 191 insertions(+), 14 deletions(-) diff --git a/cldk/models/go/__init__.py b/cldk/models/go/__init__.py index c989e80d..3812dd2e 100644 --- a/cldk/models/go/__init__.py +++ b/cldk/models/go/__init__.py @@ -15,12 +15,19 @@ ################################################################################ """Go model package — a pydantic mirror of the ``codeanalyzer-go`` (``cango``) ``analysis.json`` -(schema v2, L1/L2). +(schema v2, L1 through L4). Go is the SDK's first class-less, type-centric facade: :class:`GoType` is one model discriminated -by a ``kind`` field (``struct`` | ``interface``), never a class; package-level ``functions`` are -kept distinct from receiver ``methods``. No ``projections.py`` yet — the overview models the -Java/TS siblings ship are out of this train's SDK3 scope and add purely additively later. +by a ``kind`` field (``struct`` | ``interface`` | ``defined`` | ``alias``), never a class; +package-level ``functions`` are kept distinct from receiver ``methods``. No ``projections.py`` yet +— the overview models the Java/TS siblings ship are out of this train's SDK3 scope and add purely +additively later. + +The dataflow substrate (python-sdk#436) is the L3 intraprocedural graphs +(:class:`GoCfgEdge` / :class:`GoCdgEdge` / :class:`GoDdgEdge` on a callable) plus the L4 +interprocedural layer: :class:`GoSummaryEdge` per call site, :class:`GoParamEdge` +(``param_in``/``param_out``) and :class:`GoExternalSymbol` stand-ins on the application, and +:class:`GoDegraded` on the envelope. """ from .models import ( @@ -30,12 +37,19 @@ GoBodyNode, GoCallable, GoCallGraphEdge, + GoCdgEdge, + GoCfgEdge, + GoDdgEdge, + GoDegraded, + GoExternalSymbol, GoField, GoImport, GoMetrics, GoModule, GoParam, + GoParamEdge, GoSpan, + GoSummaryEdge, GoType, ) @@ -46,11 +60,18 @@ "GoBodyNode", "GoCallable", "GoCallGraphEdge", + "GoCdgEdge", + "GoCfgEdge", + "GoDdgEdge", + "GoDegraded", + "GoExternalSymbol", "GoField", "GoImport", "GoMetrics", "GoModule", "GoParam", + "GoParamEdge", "GoSpan", + "GoSummaryEdge", "GoType", ] diff --git a/cldk/models/go/models.py b/cldk/models/go/models.py index de6e9151..673d5879 100644 --- a/cldk/models/go/models.py +++ b/cldk/models/go/models.py @@ -145,15 +145,33 @@ class GoMetrics(_Base): class GoBodyNode(_NullSafeBase): """One entry of a callable's ``body{}`` map, keyed by local id (``L:C`` or ``@tag``). - At L1/L2 the only ``kind`` is ``call`` (a call site). ``callee`` is the one sanctioned ``null`` - on the wire — unresolved at L1, refined to a ``can://`` id at L2. ``is_goroutine`` marks a - ``go f()`` spawn. + ``kind`` is open, and widens with the analysis level: L1/L2 emit only ``call`` (a call site); + L3 adds ``statement``/``entry``/``exit``/``branch``/``return``; L4 adds the four vertex kinds + ``formal_in``/``formal_out``/``actual_in``/``actual_out``. ``callee`` is the one sanctioned + ``null`` on the wire — unresolved at L1, refined to a ``can://`` id at L2. ``is_goroutine`` + marks a ``go f()`` spawn and ``is_deferred`` a ``defer f()``. + + ``id`` is the node's absolute ``can://`` address: the owning callable's id with the map key + appended. **Never recover the callable from it by splitting on the first ``@``** — an external + callable's own id contains ``/@external/`` before the vertex ``@`` + (``can://app/app/@external/bytes/Buffer.Read@formal_in:0``), so the first ``@`` is the external + marker, not the vertex separator. Strip the known callable-id prefix instead; that is what + :func:`~cldk.analysis.go.ids.body_key` does (spec §7, A22). + + ``of`` is what an L4 vertex is a vertex *of*: a parameter name for a ``formal_in``, or + ````/``:N`` for a ``formal_out``. It — **not** ``GoCallable.parameters[]`` — is + how a formal resolves, because ``formal_in:0`` is the *receiver* on a method, which never + appears in ``parameters`` (spec §7, A9). """ + id: Optional[str] = None # L3+ (absent at L1/L2) kind: str span: Optional[GoSpan] = None callee: Optional[str] = None + of: Optional[str] = None # L4 vertices: the parameter / return slot this vertex is of + parent: Optional[str] = None # L3: enclosing body-node key is_goroutine: bool = False + is_deferred: bool = False class GoCallGraphEdge(_Base): @@ -165,6 +183,60 @@ class GoCallGraphEdge(_Base): weight: int = 1 +# ---------------------------------------------------------------------------------------------- +# L3 intraprocedural edges (``cfg`` / ``cdg`` / ``ddg`` on a callable) and the L4 ``summary`` +# ---------------------------------------------------------------------------------------------- + + +class GoCfgEdge(_Base): + """A control-flow edge between two body-node keys of one callable. + + ``kind`` is open; v0.3.0 emits ``break``, ``continue``, ``defer_resume``, ``exception``, + ``fallthrough``, ``false``, ``loop_back``, ``return``, ``switch_case`` and ``true``. It + defaults to ``fallthrough`` to match :class:`~cldk.models.python.CfgEdge`, so an emitter that + omits the unconditional case stays readable. + """ + + src: str + dst: str + kind: str = "fallthrough" + + +class GoCdgEdge(_Base): + """A control-dependence edge. The (``src``, ``dst``) pair is the whole edge — a control + dependence carries nothing else, and needs nothing else to be unique.""" + + src: str + dst: str + + +class GoDdgEdge(_Base): + """A data-dependence edge, carrying the variable it flows and how it was derived. + + ``prov`` is an open set of provenance tokens; v0.3.0 emits ``reaching-defs`` (syntactic, L3), + ``ssa`` and ``points-to`` (the L4 semantic delta, spec §4.10). + """ + + src: str + dst: str + var: Optional[str] = None + prov: List[str] = [] + + +class GoSummaryEdge(_Base): + """A per-call-site summary edge (L4, spec §4.9): the shortcut from an ``actual_in`` of a call + to the ``actual_out`` it reaches *through* the callee, so a walk can cross a call without + descending into it. + + Known gap: global → global summary edges are omitted (codellm-devkit/codeanalyzer-go#30). + Reachability is unaffected, but a depth-bounded walk may stop short where the shortcut is gone. + """ + + src: str + dst: str + var: Optional[str] = None + + # ---------------------------------------------------------------------------------------------- # Callable # ---------------------------------------------------------------------------------------------- @@ -186,23 +258,38 @@ class GoCallable(_NullSafeBase): source_file: Optional[str] = None body: Dict[str, GoBodyNode] = {} callables: Dict[str, "GoCallable"] = {} + # L3 overlays: the three intraprocedural graphs, over this callable's ``body`` keys. ``None`` + # (not ``[]``) below -a 3, so "the analyzer did not compute this" stays distinguishable from + # "it computed it and there are no edges" — the distinction ``_require_dataflow`` gates on. + cfg: Optional[List[GoCfgEdge]] = None + cdg: Optional[List[GoCdgEdge]] = None + ddg: Optional[List[GoDdgEdge]] = None + # L4 overlay: per-call-site actual_in → actual_out shortcuts. + summary: Optional[List[GoSummaryEdge]] = None def __hash__(self) -> int: return hash(self.signature) # ---------------------------------------------------------------------------------------------- -# Type — one model with a ``kind`` field (struct | interface); never a class. +# Type — one model with a ``kind`` field; never a class. # ---------------------------------------------------------------------------------------------- class GoType(_NullSafeBase): - """A Go ``struct`` or ``interface``. The two kinds carry the same wire shape, so this is one - model discriminated by ``kind`` rather than a ``Union`` of subclasses. ``base_types`` are - embedded types; ``fields`` are the struct's data fields (empty for an interface).""" + """A named Go type. All four kinds carry the same wire shape, so this is one model + discriminated by ``kind`` rather than a ``Union`` of subclasses. ``base_types`` are embedded + types; ``fields`` are the struct's data fields (empty for every other kind). + + The four kinds the analyzer emits (``symbol_table.go:432``): ``struct``, ``interface``, + ``defined`` (``type Celsius float64``) and ``alias`` (``type Alias = int``). This is a + ``Literal`` rather than an open ``str`` so a *new* kind is a loud failure and not a silent + mis-read — ``extra="ignore"`` does not cover a ``Literal`` mismatch, which is exactly why the + narrower spelling refused to parse real ``-a 1`` output (python-sdk#436). + """ id: str - kind: Literal["struct", "interface", "defined"] + kind: Literal["struct", "interface", "defined", "alias"] span: Optional[GoSpan] = None base_types: List[str] = [] callables: Dict[str, GoCallable] = {} @@ -234,9 +321,61 @@ class GoModule(_NullSafeBase): # ---------------------------------------------------------------------------------------------- +class GoExternalSymbol(_NullSafeBase): + """A call target outside the project (library member / builtin), homed on the application as + ``/@external//`` and keyed by that id. + + Richer than its TypeScript counterpart: at L4 the analyzer synthesizes a **stand-in body** for + a library callable so a flow can pass *through* it — ``body`` holds that callable's + ``formal_in``/``formal_out`` vertices, and ``receiver`` the method's receiver (which is + ``formal_in:0``, and is *not* in ``parameters``; spec §6, §7 A9). + """ + + id: str + kind: str = "external" + name: str + module: str + signature: Optional[str] = None + parameters: List[GoParam] = [] + return_type: Optional[str] = None + receiver: Optional[GoParam] = None + body: Dict[str, GoBodyNode] = {} + + +class GoParamEdge(_Base): + """An L4 ``param_in`` / ``param_out`` edge: the interprocedural binding between a call site's + ``actual_in``/``actual_out`` vertex and the callee's ``formal_in``/``formal_out``. Endpoints + are absolute ``can://`` ids (unlike the intraprocedural edges, which are callable-local keys), + because the two ends live in different callables.""" + + src: str + dst: str + var: Optional[str] = None + + +class GoDegraded(_NullSafeBase): + """One degradation marker: an L4 step that fell back to a coarser result (spec §4.15). + + ``code`` is an open vocabulary of plain strings (``range_func``, ``unmapped_ssa``, ``no_vta``, + ``ssa_build``, …). **This is the honesty signal**: a non-empty ``GoAnalysis.degraded`` means + part of the graph is approximated, so the SDK must not certify *absence* of flow over it — a + returned path is still a real may-flow, but "no path" is not a proof. + """ + + code: str + detail: Optional[str] = None + packages: List[str] = [] + callables: List[str] = [] + + class GoApplication(_NullSafeBase): - """The application root: the containment tree (``symbol_table``) plus the one app-scope overlay - the analyzer emits at L2 (``call_graph``).""" + """The application root: the containment tree (``symbol_table``) plus the app-scope overlays — + ``call_graph`` at L2, and at L4 the interprocedural parameter bindings + (``param_in``/``param_out``) and the library stand-ins (``external_symbols``). + + Note ``degraded`` is **not** here: it sits on the :class:`GoAnalysis` envelope beside + ``application``, which is where ``internal/schema/v2/schema.go:63`` puts it. + """ application_name: str identity_namespace: str @@ -245,6 +384,9 @@ class GoApplication(_NullSafeBase): kind: Literal["application"] = "application" symbol_table: Dict[str, GoModule] call_graph: List[GoCallGraphEdge] = [] + param_in: List[GoParamEdge] = [] + param_out: List[GoParamEdge] = [] + external_symbols: Dict[str, GoExternalSymbol] = {} class GoAnalyzer(_Base): @@ -262,6 +404,20 @@ class GoAnalysis(_Base): k_limit: Optional[int] = None # L3+ (absent at L1/L2) analyzer: GoAnalyzer application: GoApplication + #: L4 degradation markers; ``[]`` when the run degraded nothing and below ``-a 4``. Envelope + #: scope, not application scope — see :class:`GoApplication`. + degraded: List[GoDegraded] = [] + + @model_validator(mode="before") + @classmethod + def _coerce_null_degraded(cls, data): + # The envelope omits ``degraded`` entirely when nothing degraded, but a Go emitter that + # marshalled a nil slice would send ``null``, which ``List[GoDegraded]`` rejects. The + # containment models get this from ``_NullSafeBase``; the envelope is a plain ``_Base`` + # (every other field is required or scalar), so it coerces its one collection here. + if isinstance(data, dict) and data.get("degraded") is None: + data["degraded"] = [] + return data # Resolve the forward reference in the self-recursive callable. From 8d2f18deb99343a11cc5d5f2509808d96b9c2a06 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 5 Nov 2026 11:20:56 -0500 Subject: [PATCH 02/15] feat(go): declare the L3/L4 query surface and Go's id grammar The fifteen accessors with PythonAnalysisBackend's names and signatures, so one caller's dataflow code ports between languages. taint() is concrete on the ABC -- both backends share the policy rather than reimplementing it -- and adds a Go-specific rule: a non-empty degraded[] voids every exhausted certification, since absence of a path is not a proof over an approximated graph. Also declares the Java-style port-connectivity gate. ids.py isolates the four traps where a shared helper would be wrong for Go rather than merely absent, verified on cango 0.3.0 output: - a body-node id must never be split on its first @ -- a stand-in's id holds /@external/ before the vertex @, and the naive partition mis-owns all 129 external vertices in testdata/dataflow; - a formal resolves by its own `of`, not by ordinal into parameters[], since a method's formal_in:0 is the receiver (of == parameters[0] on only 174/276 callables); - a global's module half is slash-separated, so the spelling a Go programmer writes (libflow.Last) failed dot-wise segment_match on all 23 globals; - degraded[] is the honesty signal, not a refusal. Refs #436 --- cldk/analysis/go/backend.py | 504 +++++++++++++++++++++++++++++++++++- cldk/analysis/go/ids.py | 203 +++++++++++++++ 2 files changed, 706 insertions(+), 1 deletion(-) create mode 100644 cldk/analysis/go/ids.py diff --git a/cldk/analysis/go/backend.py b/cldk/analysis/go/backend.py index 476ed680..cfa3684b 100644 --- a/cldk/analysis/go/backend.py +++ b/cldk/analysis/go/backend.py @@ -42,13 +42,31 @@ from __future__ import annotations from abc import abstractmethod -from typing import ClassVar, Dict, List +from typing import ClassVar, Collection, Dict, List, Mapping, Sequence, Tuple from cldk.analysis.commons.backend import AnalysisBackend +from cldk.analysis.commons.bounds import ( + DEFAULT_DEPTH, + DEFAULT_MAX_NODES, + DEFAULT_MAX_PATHS, + DEFAULT_PAGE_SIZE, + EdgeOrder, + check_depth, + check_max_paths, + reject_bare_string, +) +from cldk.analysis.commons.graphs import edge_sort_key, sdg_rel_pattern, sdg_rels, taint_verdict, via_table +from cldk.analysis.commons.resolve import resolve_sanitizers +from cldk.analysis.commons.results import Diagnostic, EdgePage, FlowPath, FlowPaths, Slice, SliceNode, TaintResult from cldk.models.go import ( GoApplication, GoCallable, GoCallGraphEdge, + GoCdgEdge, + GoCfgEdge, + GoDdgEdge, + GoDegraded, + GoExternalSymbol, GoField, GoModule, GoParam, @@ -64,6 +82,48 @@ "of {app!r}; query the Neo4j backend, which carries it" ) +#: Raised by :meth:`GoAnalysisBackend._require_connected_ports` when this application's +#: ``formal_in`` vertices carry no outgoing dependence edge: the interprocedural layer exists on the +#: wire but nothing crosses into it, so every slice and every path over a port would come back empty +#: and an empty there is indistinguishable from a proved absence. Mirrors the Java backend's +#: ``PORTS_DISCONNECTED`` (``java/backend.py``), down to being decided from the *data* rather than +#: from an analyzer version string. +PORTS_DISCONNECTED = ( + "{accessor} needs the interprocedural ports of {app!r} to carry dependence, and no formal_in " + "vertex in this analysis has an outgoing dependence edge — so every answer would be empty for " + "want of edges rather than for want of a flow. Re-emit at -a 4 (`cango -a 4`, or " + "`--emit neo4j`, which is always full depth)." +) + +_CFG_KEY, _CDG_KEY, _DDG_KEY = edge_sort_key("cfg"), edge_sort_key("cdg"), edge_sort_key("ddg") + + +def cfg_sort_key(edge: GoCfgEdge) -> Tuple: + """The canonical order for :meth:`GoAnalysisBackend.get_cfg`: source, target, kind.""" + return _CFG_KEY(edge) + + +def cdg_sort_key(edge: GoCdgEdge) -> Tuple: + """The canonical order for :meth:`GoAnalysisBackend.get_cdg`: source, target.""" + return _CDG_KEY(edge) + + +def ddg_sort_key(edge: GoDdgEdge) -> Tuple: + """The canonical order for :meth:`GoAnalysisBackend.get_ddg`: source, target, variable, prov.""" + return _DDG_KEY(edge) + + +#: The three orders, shared verbatim with the sibling languages so a page is ordered over the ids a +#: caller sees. The Cypher column list is what the backends' ``WITH`` clause projects. +CFG_ORDER = EdgeOrder(cfg_sort_key, ("src", "dst", "coalesce(kind,'')")) +CDG_ORDER = EdgeOrder(cdg_sort_key, ("src", "dst")) +DDG_ORDER = EdgeOrder(ddg_sort_key, ("src", "dst", "coalesce(var,'')", "coalesce(prov,[])")) + +#: The five relationship types a slice follows, in Go's ``GO_`` spelling. +SDG_RELS = sdg_rels("GO") +SDG_REL_PATTERN = sdg_rel_pattern("GO") +VIA = via_table("GO") + class GoAnalysisBackend(AnalysisBackend[GoApplication, GoModule, GoType, GoCallable, GoField, GoParam]): """Abstract base every Go analysis backend implements (see the module docstring). @@ -114,3 +174,445 @@ def get_source(self, node_id: str) -> str: @abstractmethod def get_call_graph_json(self) -> str: """The application's call graph as a JSON string (Tier A).""" + + # ===================================================================================== + # L3/L4 DATAFLOW (python-sdk#436). The names and signatures are + # :class:`~cldk.analysis.python.backend.PythonAnalysisBackend`'s, deliberately — a caller who + # has written against one language's dataflow surface can point the same code at another. + # + # DEPTH DEFAULTS DIFFER BY RETURN TYPE, as on every sibling. The two *slices* and the cone + # default to :data:`~cldk.analysis.commons.bounds.DEFAULT_DEPTH` and cap ``max_nodes``: a + # bounded traversal is a complete answer to a narrower question, and ``total`` says what was + # left out. The two *predicates* and the *path* queries default to ``depth=None`` — unbounded — + # because a hop budget on a boolean is not a smaller answer but a wrong one: "no flow" and "no + # flow within five hops" would collapse into the same ``False`` with nothing to tell them apart. + # + # ANALYSIS LEVEL. ``cfg``/``cdg``/``ddg`` exist from ``cango -a 3`` and the interprocedural + # overlays from ``-a 4``. A backend attached to a shallower analysis MUST raise rather than + # return an empty page (D7). ``--emit neo4j`` is always full depth, so only the local backend + # can be below the line — which is why ``_require_dataflow`` is the *local* backend's method + # while :meth:`_require_connected_ports` below, asked of the data, is declared for both. + # + # FOUR GO-SPECIFIC TRAPS, each measured on real ``cango`` 0.3.0 output rather than assumed + # (spec `v2-l3-l4-dataflow.md` §7; the issue is python-sdk#436): + # + # 1. NEVER SPLIT A BODY-NODE ID ON ITS FIRST ``@``. An external callable's own id contains + # ``/@external/`` *before* the vertex ``@``: + # ``can://app/app/@external/bytes/Buffer.Read@formal_in:0``. Strip the known callable-id + # prefix instead — :func:`~cldk.analysis.go.ids.split_body_node_id`. + # 2. ``of``, NOT ``parameters[]``, RESOLVES A FORMAL. On a method ``formal_in:0`` is the + # *receiver*, which never appears in ``GoCallable.parameters`` — measured: ``of`` equals + # ``parameters[0].name`` on only 174 of 276 callables, and the receiver reads ``_recv`` or + # the receiver variable's own name. A closure's captures are ``formal_in`` vertices too, + # with ``of`` spelled ``:x`` and ``parameters`` empty. + # 3. A GLOBAL'S MODULE HALF IS SLASH-SEPARATED. + # :func:`~cldk.analysis.commons.resolve.value_candidate` turns ``:example.com/lib::V`` + # into the dotted ``example.com/lib.V``, so the spelling a Go programmer writes — ``lib.V`` — + # fails the dot-separated :func:`~cldk.analysis.commons.resolve.segment_match`. Go therefore + # matches a value on the package-qualified spelling as well; see + # :func:`~cldk.analysis.go.ids.go_value_names`. + # 4. ``degraded`` IS THE HONESTY SIGNAL. A non-empty :meth:`get_degraded` means part of the + # graph is approximated, so *absence* of a path is not a proof over it. The flow surface + # reports this rather than deciding for the caller: it is a :class:`Diagnostic` on the + # result's ledger, which voids ``exhausted`` — never a refusal, because a returned path is + # still a real may-flow. + # ===================================================================================== + + # -----[ addressing ]----- + @abstractmethod + def resolve_callable(self, name: str, *, in_class: str | None = None, in_module: str | None = None) -> SliceNode: + """The one callable ``name`` names, as an addressable position. + + Go accepts the receiver forms a Go programmer writes, resolved segment-wise against the + full signature by :func:`~cldk.analysis.commons.resolve.resolve_callable_signature`: + ``Query``, ``DB.Query``, ``(*DB).Query``, ``sql.DB.Query`` and + ``database/sql.DB.Query`` all name the same method where that is unambiguous. + + Args: + name: The callable's name, whole or a suffix of its signature on segment boundaries. + in_class: Disambiguate by owning *type* (Go has no class; the keyword keeps the shared + spelling so one caller's code ports between languages). + in_module: Disambiguate by module — the repo-relative ``.go`` path, or the package's + dotted name. + + Raises: + AmbiguousName: More than one callable matched; the candidates are carried. + SelectorNotInGraph: Nothing matched, or a keyword excluded every match. + """ + + @abstractmethod + def resolve_value(self, name: str, *, within: str) -> SliceNode: + """The one value of ``within`` that ``name`` names — a ``formal_in`` port vertex. + + The domain is the ``formal_in`` vertices of the resolved callable, which in Go is wider + than "its parameters": a method's receiver is ``formal_in:0``, a closure's captures are + ``formal_in`` vertices with ``of`` spelled ``:x``, and a package-level variable the + callable reads is one with ``of`` spelled ``:::``. Each is translated + out of the analyzer's vocabulary by + :func:`~cldk.analysis.commons.resolve.value_candidate`, so the returned + :class:`~cldk.analysis.commons.results.SliceNode` carries ``kind="parameter"`` / + ``"capture"`` / ``"global"`` and a readable ``name`` — never a raw ``:…`` string. + + **A global is matchable by the spelling a Go programmer writes.** ``value_candidate`` + renders the module half as the import path (``example.com/lib.V``), whose own separators are + ``/``; matching only dot-wise would refuse ``lib.V``. Go accepts the package-qualified form + as well (trap 3 above). + + Args: + name: The value's name — bare (``id``, ``_recv``), package-qualified (``lib.V``) or the + full import-path form. + within: The callable to look inside, resolved as in :meth:`resolve_callable`. + + Raises: + AmbiguousName: More than one value matched. + SelectorNotInGraph: No value of ``within`` carries that name — which is also how a + caller attached to an analysis shallower than ``-a 4`` is refused, by name, since + the port vertices do not exist there at all. + """ + + # -----[ per-callable graphs ]----- + @abstractmethod + def get_cfg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoCfgEdge]: + """One page of the control flow edges within one callable. + + Args: + callable: The callable's name, resolved by :meth:`resolve_callable`. + in_class: Disambiguate by owning type, as in :meth:`resolve_callable`. + page_size: Most edges to return. See + :data:`~cldk.analysis.commons.bounds.DEFAULT_PAGE_SIZE`. + cursor: ``next_cursor`` from a previous page; ``None`` starts at the beginning. + + Returns: + An :class:`~cldk.analysis.commons.results.EdgePage` of + :class:`~cldk.models.go.GoCfgEdge`, each carrying the analyzer's ``kind``. Go's set — + ``fallthrough``, ``true``, ``false``, ``switch_case``, ``loop_back``, ``exception``, + ``return``, ``break``, ``continue`` and the Go-specific ``defer_resume`` (the edge back + from a deferred call) — measured on ``cango`` 0.3.0. A conditional's two successors stay + two edges discriminated by ``kind``, which is why ``kind`` is in :data:`CFG_ORDER`. + Endpoints are the body nodes' own ``can://`` ids, the spelling :meth:`get_source` + accepts. + + Raises: + AmbiguousName: ``callable`` named more than one callable. + SelectorNotInGraph: Nothing matched. + ValueError: ``page_size`` below 1, or ``cursor`` not from a previous page of this + accessor and this callable. + CodeanalyzerUsageException: (local backend) built below ``-a 3``. + """ + + @abstractmethod + def get_cdg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoCdgEdge]: + """One page of the control dependence edges within one callable. + + ``src`` is the branching node ``dst`` is control dependent on — post-dominance over the CFG + :meth:`get_cfg` returns, computed by the analyzer and not re-derived here. Arguments, bounds + and failures are :meth:`get_cfg`'s; the order is :data:`CDG_ORDER`. + """ + + @abstractmethod + def get_ddg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoDdgEdge]: + """One page of the data dependence edges within one callable. + + Each edge carries the variable it flows (``var``) and its evidence (``prov``). + + **Go's DDG has all three provenance tiers**, unlike TypeScript's single one: ``cango`` + emits ``reaching-defs`` (the syntactic L3 pass), and at ``-a 4`` adds ``ssa`` and + ``points-to`` — the semantic delta of spec §4.10. So + :func:`~cldk.analysis.commons.results.prov_rank`'s certainty ranking is live here, and a + caller comparing two hops' evidence gets a real ordering. + + Arguments, bounds and failures are :meth:`get_cfg`'s; the order is :data:`DDG_ORDER`, which + includes ``var`` and ``prov`` because one statement pair legitimately appears more than once + carrying different variables, and collapsing those would drop dependences. + """ + + # -----[ slicing and reachability ]----- + @abstractmethod + def slice_backward(self, src: str, *, within: str, depth: int | None = DEFAULT_DEPTH, max_nodes: int = DEFAULT_MAX_NODES) -> Slice: + """Everything the value ``src`` depends on: reverse reachability over the SDG. + + The edge set is :data:`SDG_RELS` — data and control dependence within a callable, the two + parameter-passing relationships across a call, and the callee summaries at a call site. All + five point *with* the flow, so a backward slice follows them reversed. + + ``within`` is required: a value name is scoped by its callable and :meth:`resolve_value` + cannot resolve one without it, so a ``None`` default would be a signature that raises on + its own default. + + Args: + src: The value's name, resolved by :meth:`resolve_value`. + within: The callable to look inside, resolved as in :meth:`resolve_callable`. + depth: Most hops from the seed; ``None`` for the whole cone. + max_nodes: Most nodes in the result. A cap that fires is reported by + :attr:`~cldk.analysis.commons.results.Slice.complete`, with + :attr:`~cldk.analysis.commons.results.Slice.total` the full size. + + Raises: + AmbiguousName / SelectorNotInGraph: as :meth:`resolve_value`. + ValueError: ``depth`` is not a positive ``int``, or ``max_nodes`` is below 1. + CodeanalyzerExecutionException: this analysis's ports carry no dependence + (:data:`PORTS_DISCONNECTED`). + """ + + @abstractmethod + def slice_forward(self, src: str, *, within: str, depth: int | None = DEFAULT_DEPTH, max_nodes: int = DEFAULT_MAX_NODES) -> Slice: + """Everything the value ``src`` can affect: forward reachability over :data:`SDG_RELS`. + + Arguments, bounds and failures are :meth:`slice_backward`'s, followed with the flow rather + than against it. + """ + + @abstractmethod + def backward_cone(self, sinks: Sequence[str], *, depth: int | None = DEFAULT_DEPTH, max_nodes: int = DEFAULT_MAX_NODES) -> Slice: + """Every callable that can reach any of these sinks, over the **call graph**. + + The coarse counterpart of :meth:`slice_backward`: a cone of *callables*, not of values, so + the sinks are resolved by :meth:`resolve_callable` and the answer is what a triage pass + narrows to before asking for a value-level path. + + Args: + sinks: Callable names, each resolved by :meth:`resolve_callable`. + depth: Most call hops; ``None`` for every ancestor. + max_nodes: Most nodes in the result, reported as in :meth:`slice_backward`. + """ + + @abstractmethod + def reaches(self, src: str, dst: str, *, depth: int | None = None) -> bool: + """Is there a call path from the callable ``src`` to the callable ``dst``? + + Over the same edge set :meth:`get_callers` / :meth:`get_callees` expose, so the boolean + cannot disagree with the neighbours a caller can enumerate. ``depth`` defaults to ``None``: + a hop budget on a boolean makes ``False`` ambiguous (see the block comment above). + ``reaches(x, x)`` is the *cycle* question — ``True`` only if a real path returns to ``x``. + """ + + @abstractmethod + def paths_between( + self, src: str, dst: str, *, src_within: str, dst_within: str, depth: int | None = None, max_paths: int = DEFAULT_MAX_PATHS + ) -> FlowPaths: + """How the value ``src`` reaches the value ``dst`` — the witnesses, shortest first. + + **This proves a flow; it does not refute one.** An empty result means "no path was found", + which a caller cannot tell from "the flow left the resolved graph". :meth:`taint` is the + accessor that reports that distinction per pair, and it is what triage should call. + + Args: + src / src_within: The source value and its callable, as :meth:`resolve_value`. + dst / dst_within: The sink value and its callable. + depth: Most hops; ``None`` (the default) for unbounded. + max_paths: Most witnesses to return. Truncation is reported by + :attr:`~cldk.analysis.commons.results.FlowPaths.complete`. + + Raises: + ValueError: ``src`` and ``dst`` resolve to the same position + (:func:`~cldk.analysis.commons.bounds.check_distinct_endpoints`), ``depth`` is not + positive, or ``max_paths`` is below 1. + """ + + @abstractmethod + def flows_to_call(self, src: str, callee: str, *, within: str, depth: int | None = None) -> bool: + """Does the value ``src`` reach **any** value entering a call to ``callee``? + + The coarse taint question: "does my request parameter reach ``DB.Query`` at all", without + naming which argument. ``callee`` is resolved by :meth:`resolve_callable`, so the Go + receiver forms of that method apply here too. + """ + + @abstractmethod + def flows_to_argument(self, src: str, callee: str, arg: str, *, within: str, depth: int | None = None) -> bool: + """Does the value ``src`` reach ``callee``'s parameter ``arg`` specifically? + + :meth:`flows_to_call` narrowed to one port: ``arg`` is resolved by :meth:`resolve_value` + *within* ``callee``, so a Go receiver is addressable as ``arg`` too. + """ + + # -----[ the honesty signals ]----- + @abstractmethod + def get_external_symbols(self) -> Dict[str, GoExternalSymbol]: + """The library stand-ins this analysis synthesized, keyed by ``can://…/@external/…`` id. + + At ``-a 4`` the analyzer gives a library callable a stand-in body — ``formal_in`` / + ``formal_out`` vertices and a ``receiver`` — so a flow can pass *through* ``bytes.Buffer.Read`` + or ``database/sql.DB.Query`` rather than stopping at the project boundary (spec §6). This is + where a path's external hops are explained; it is not a dependency list. + """ + + @abstractmethod + def get_degraded(self) -> List[GoDegraded]: + """The degradation markers this analysis carries — empty when nothing was approximated. + + **Read this before trusting an absence.** A non-empty list means some L4 step fell back to + a coarser result (``range_func``, ``unmapped_ssa``, ``no_vta``, …), so a path that *is* + returned is still a real may-flow, but "no path" is not a proof over the affected packages + and callables. :meth:`taint` reports this on its own ledger rather than making the judgement + for the caller; this accessor is how a caller inspects it directly. + """ + + @abstractmethod + def _require_connected_ports(self, accessor: str) -> None: + """Refuse ``accessor`` when this application's ports carry no dependence. + + Named after the Java backend's method of the same name and for the same reason: the + interprocedural layer can be *present* on the wire and still have nothing crossing into it, + in which case every slice and every path over a port is empty for want of edges rather than + for want of a flow — indistinguishable from a proved absence, which is the one answer this + surface must never fake. + """ + + @property + @abstractmethod + def _ports_carry_dependence(self) -> bool: + """Whether **this** application's ``formal_in`` vertices have any outgoing SDG edge. + + Asked of the data, once per backend, rather than hard-coded from an analyzer version: the + refusal above is a statement about what was emitted, so a graph that connects the two layers + must make these accessors work with no change here. + """ + + # -----[ taint: the one concrete method, so both backends cannot disagree on policy ]----- + def taint( + self, + sources: Sequence[Tuple[str, str]], + sinks: Sequence[Tuple[str, str]], + sanitizers: Sequence[Tuple[str, str] | str] = (), + *, + depth: int | None = None, + max_paths: int = DEFAULT_MAX_PATHS, + ) -> TaintResult: + """Which of these sources reach which of these sinks, and what to make of the ones that do not. + + The verb triage needs. :meth:`paths_between` *proves* a flow; a caller who gets ``[]`` back + from it cannot tell "no flow exists" from "the flow left the resolved graph". ``taint()`` + runs m sources against n sinks in **one** traversal and reports that distinction per pair: + the witnesses in :attr:`~cldk.analysis.commons.results.FlowPaths.paths`, the pairs an + absence claim can be built on in + :attr:`~cldk.analysis.commons.results.TaintResult.exhausted`, and everything else in + ``unresolved``. + + **Sources, sinks and sanitizers are the caller's to supply.** This SDK ships no Go framework + catalogue and derives no default set: a per-language vocabulary of taint sources is policy + that rots, and this accessor is the mechanism. + + **A sanitizer is two mechanisms wearing one word**, told apart by shape. A bare ``str`` cuts + a *callable* on the path — what a transforming sanitizer is. A ``(name, within)`` pair cuts a + *variable* inside that callable, which is the only thing that severs a *validating* guard, + because a guard never appears on the data path at all. Both cuts are applied **inside** the + search, not to the rows it returns: filtering afterwards would report nothing for a source + whose ten shortest paths are sanitized and whose eleventh is not — a false refutation, the + one output this accessor must never produce. + + **Three rules, shared verbatim with the sibling languages (leg 4b, #382):** an explicit + ``depth`` empties ``exhausted`` by rule — a pair with no path inside five hops is not + refuted, it is unmeasured; ``complete`` is the *batch's* flag, so one blocked pair voids + every absence claim in the result; and ``max_paths`` caps per pair, never per call. + + **Go adds a fourth: a degraded analysis cannot certify an absence.** When + :meth:`get_degraded` is non-empty, part of the graph is approximated, so every pair's + ``exhausted`` certification is voided by a diagnostic on the ledger. Not a refusal — a + returned path over a degraded graph is still a real may-flow, and refusing would throw away + the findings that matter most. + + **The level gate belongs to the walk**, not here: ``_require_dataflow`` is a local backend's + method and a graph backend has no shallow mode to guard. :meth:`_taint_walk` opens with it. + + Args: + sources: The values taint enters at, each ``(name, within)`` — :meth:`resolve_value`'s + addressing. + sinks: The values it must not reach, addressed the same way. + sanitizers: Bare names cut callables; ``(name, within)`` pairs cut variables. + depth: Most hops a path may take; ``None`` (the default) for no bound, because a bound + turns a refutation into an artefact of the budget. + max_paths: Most witnesses **per pair**. + + Returns: + A :class:`~cldk.analysis.commons.results.TaintResult`. + + Raises: + AmbiguousName: A name, or a sanitizer's ``within``, matched more than one thing. + SelectorNotInGraph: A name matched nothing, or a sanitizer's shape disagrees with what + it resolves to. + TypeError: ``sources`` or ``sinks`` is a bare string, which would unpack into a pair. + ValueError: ``depth`` is not a positive ``int``, ``max_paths`` is below 1, or ``sources`` + / ``sinks`` is empty (refused, not answered ``[]``). + CodeanalyzerExecutionException: this analysis's ports carry no dependence. + """ + # The order of the checks is part of the contract, cheapest-to-be-wrong-about first: the + # bounds before any name, the names before any sanitizer, the sanitizers before the walk. + check_depth(depth) + check_max_paths(max_paths) + reject_bare_string("sources", sources) + reject_bare_string("sinks", sinks) + if not sources: + raise ValueError("sources= names nothing to taint from; pass at least one (name, within) pair") + if not sinks: + raise ValueError("sinks= names nothing to taint to; pass at least one (name, within) pair") + self._require_connected_ports("taint") + srcs = [self.resolve_value(name, within=within) for name, within in sources] + dsts = [self.resolve_value(name, within=within) for name, within in sinks] + cuts, cut_callables = resolve_sanitizers(sanitizers, resolve_callable=self.resolve_callable, edge_vars_in=self._edge_vars_in) + rows, blocked = self._taint_walk(srcs, dsts, cuts=cuts, cut_callables=cut_callables, depth=depth, max_paths=max_paths) + # A degraded analysis cannot certify an absence (trap 4). The diagnostic is filed under + # *every* requested pair, not under one key of this method's devising: ``taint_verdict`` + # reads the keys to decide ``exhausted``, and a key no requested pair claims would cost + # every pair its certification for want of anything able to attribute it. + degraded = self.get_degraded() + if degraded: + codes = sorted({d.code for d in degraded}) + note = Diagnostic( + code="analysis_degraded", + message=( + f"this analysis carries {len(degraded)} degradation marker(s) ({', '.join(codes)}), so part of the " + "SDG is approximated: a path returned here is a real may-flow, but the absence of one is not a " + "proof. See get_degraded() for the affected packages and callables." + ), + ) + per_pair = {(a.ref, b.ref): [note] for a in srcs for b in dsts} + blocked = {k: list(blocked.get(k, [])) + v for k, v in per_pair.items()} | {k: v for k, v in blocked.items() if k not in per_pair} + return taint_verdict(sources, sinks, srcs, dsts, rows=rows, blocked=blocked, depth=depth, max_paths=max_paths) + + @abstractmethod + def _taint_walk( + self, + srcs: Sequence[SliceNode], + dsts: Sequence[SliceNode], + *, + cuts: List[Dict[str, str]], + cut_callables: List[str], + depth: int | None, + max_paths: int, + ) -> Tuple[List[Tuple[str, str, FlowPath]], Mapping[Tuple[str, str], List[Diagnostic]]]: + """The sanitized shortest walks between every source and every sink, and what stopped a pair. + + Rows are ``(source ref, sink ref, path)`` triples, shortest-first within a pair and in + :func:`~cldk.analysis.commons.graphs.hop_sort_key` order among equals, capped at + ``max_paths + 1`` **per pair** — the extra row is what lets :meth:`taint` report truncation + without a second counting traversal. Grouping and trimming are :meth:`taint`'s. + + The second element is the frontier ledger, keyed by the ``(source ref, sink ref)`` pair it + implicates. The key is the association, not the message: ``exhausted`` is decided from these + keys, and recovering a pair by parsing prose out of a :class:`Diagnostic` would resurrect + the derivation the key exists to avoid. A condition that affects several pairs is filed + under *each* of them. + + A local backend opens with ``self._require_dataflow()``: a graph backend has no analysis + level to measure, so the gate lives in the implementation that can answer it. + """ + + @abstractmethod + def _edge_vars_in(self, callable_id: str) -> Collection[str]: + """The variable names carried by SDG edges scoped to this callable — the domain a variable + sanitizer is checked against. + + Not :meth:`resolve_value`, which addresses ``formal_in`` port vertices only: most real edge + variables are locals, so validating a sanitizer through the resolver would refuse a + legitimate one for not being a parameter. + """ + + @abstractmethod + def describe(self, nodes: Sequence[object]) -> List[SliceNode]: + """Fill in :attr:`~cldk.analysis.commons.results.SliceNode.source` for these positions. + + A second call because a slice answers *where* and source answers *what*, and source is the + one field with no size ceiling — so the traversals return ``source=None`` and a caller + hydrates the handful of nodes it decided to read. + """ diff --git a/cldk/analysis/go/ids.py b/cldk/analysis/go/ids.py new file mode 100644 index 00000000..82b57d44 --- /dev/null +++ b/cldk/analysis/go/ids.py @@ -0,0 +1,203 @@ +################################################################################ +# Copyright IBM Corporation 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +"""Go's id and name grammar — the four traps of spec §7 (python-sdk#436), in one place. + +Each function here exists because a rule that holds for Python or TypeScript does **not** hold for +Go, and the shared helpers in :mod:`cldk.analysis.commons` would therefore give a wrong answer +rather than an error. They are separated from both backends so the local and the Neo4j backend +cannot drift on them: a body-node id split one way in one backend and another way in the other is +precisely the class of bug a parity contract test cannot see, because both backends would be +*consistently* wrong about the same external callable. +""" + +from __future__ import annotations + +from typing import Iterable, List, Sequence, Tuple + +from cldk.analysis.commons.resolve import value_candidate + +#: The marker the analyzer puts in a library stand-in's **callable** id, before any vertex ``@``: +#: ``can:////@external/bytes/Buffer.Read``. Its leading ``@`` is why +#: :func:`split_body_node_id` may not split on the first one. +EXTERNAL_MARKER = "/@external/" + + +def body_node_id(callable_id: str, body_key: str) -> str: + """The analyzer's own global id for one of a callable's body nodes. + + The emitter's rule, adopted rather than re-derived — a synthetic key owns a leading ``@`` + (``"@entry"``, ``"@formal_in:1"``) and a statement key does not (``"10:25"``), so joining with + an unconditional ``"@"`` would mint ``…Add@@formal_in:0``, which names nothing in ``body`` and + nothing in the graph. Verified against real ``cango`` 0.3.0 output: every ``body`` entry's own + ``id`` field equals this function applied to its map key. + + Identical in form to Python's ``body_node_id``; it is restated here rather than imported + because :func:`split_body_node_id` — its inverse — is *not* shared, and keeping the pair + together is what makes the asymmetry visible. + """ + return f"{callable_id}{body_key}" if body_key.startswith("@") else f"{callable_id}@{body_key}" + + +def split_body_node_id(node_id: str, callable_ids: Iterable[str]) -> Tuple[str, str] | None: + """``(callable id, body key)`` for ``node_id``, or ``None`` if no known callable owns it. + + **Trap 1 of spec §7, and the reason this function is not a one-line partition.** The obvious + inverse of :func:`body_node_id` is ``node_id.partition("@")`` — which is what the Python + backend can afford, because a Python callable id contains no ``@``. A Go library stand-in's id + does:: + + can://dataflow/dataflow/@external/bytes/Buffer.Read@formal_out:0 + ^ first @ is the external marker, not the vertex separator + + Splitting there yields the callable ``can://dataflow/dataflow/`` and the key + ``external/bytes/Buffer.Read@formal_out:0``: two strings that name nothing, handed onward as if + they did. So the owner is found by **stripping a known callable-id prefix** instead, longest + first — the longest match is the right one because a callable id can be a proper prefix of a + nested closure's (``…flow.Closure`` and ``…flow.Closure.closure_1``), and the shorter would + claim the longer's vertices. + + Args: + node_id: A body node's absolute ``can://`` id. + callable_ids: Every callable id in the application, stand-ins included. Order is + irrelevant; this sorts. + + Returns: + ``(callable_id, body_key)`` where ``body_node_id(*result) == node_id``, or ``None`` when no + candidate owns it — a miss is returned, not raised, because the callers here are describing + a walk's nodes and an unknown node is a node to skip, not an error to propagate. + """ + for cid in sorted(callable_ids, key=len, reverse=True): + if not node_id.startswith(cid) or node_id == cid: + continue + rest = node_id[len(cid) :] + # The join inserted a bare "@" only for a statement key; a synthetic key brought its own. + if rest.startswith("@@"): + return cid, rest[1:] + if rest.startswith("@"): + # Either a synthetic key whose "@" is its own, or a statement key whose "@" was the + # join. Both spellings are tried by the callers against the real ``body`` map, which is + # the only thing that can tell "@formal_in:0" from a joined "formal_in:0"; what matters + # here is that the *callable* half is right, and it is. + return cid, rest + return None + + +def go_value_names(var: str) -> List[str]: + """Every spelling a caller may use for the value whose analyzer ``var`` is ``var``. + + **Trap 3 of spec §7.** :func:`~cldk.analysis.commons.resolve.value_candidate` renders a Go + global's ``var`` — ``:example.com/dataflow/libflow::Last`` — as the dotted + ``example.com/dataflow/libflow.Last``. That is the right *display* name, but the separators + inside its module half are ``/``, and + :func:`~cldk.analysis.commons.resolve.segment_match` narrows on ``.`` — so the spelling a Go + programmer actually writes, ``libflow.Last``, matches **nothing**:: + + segment_match("libflow.Last", "example.com/dataflow/libflow.Last") -> False + segment_match("Last", "example.com/dataflow/libflow.Last") -> True (bare leaf) + + Verified on all 28 global ports of the ``testdata/dataflow`` fixture. Rather than loosen + ``segment_match`` for every language, Go adds the package-qualified spelling as a second name + for the same value. A parameter or a capture has exactly one name, so this is the identity + there and costs nothing. + + The names are what a resolver matches against; the **first** is the canonical one + :func:`~cldk.analysis.commons.resolve.value_candidate` chose, so an error message and a + returned :class:`~cldk.analysis.commons.results.SliceNode` still read in one vocabulary. + """ + v = value_candidate(var) + if v.kind != "global" or not v.defined_in: + return [v.name] + package = v.defined_in.rsplit("/", 1)[-1] + qualified = f"{package}.{v.leaf}" + return [v.name] if qualified == v.name else [v.name, qualified] + + +def receiver_of(body: dict) -> str | None: + """The ``of`` of this callable's ``formal_in:0``, or ``None``. + + **Trap 2 of spec §7, stated as the thing it protects.** A Go method's ``formal_in:0`` is its + *receiver*, which never appears in ``GoCallable.parameters`` — so a backend that paired + ``formal_in:N`` with ``parameters[N]`` would name every one of a method's formals one position + off, silently. Measured on the ``testdata/dataflow`` fixture: ``formal_in:0.of`` equals + ``parameters[0].name`` on only 174 of 276 callables, and on a method it reads ``_recv`` or the + receiver variable's own name while ``parameters`` is empty or starts at the first real argument. + + The rule the backends follow instead: **resolve a formal by its own ``of``, never by ordinal + into ``parameters``.** This accessor exists for the one case where the receiver specifically is + wanted; everything else reads ``of`` directly off the vertex. + """ + node = body.get("@formal_in:0") + return getattr(node, "of", None) if node is not None else None + + +def is_external(node_id: str) -> bool: + """Is this id (or any vertex of it) a library stand-in rather than project code? + + The test is for :data:`EXTERNAL_MARKER`, not for a position within the id: the marker sits in + the *callable* half, so it answers the same for a stand-in's callable id and for any of its + vertex ids. + """ + return EXTERNAL_MARKER in node_id + + +def callable_match_names(signature: str) -> Tuple[str, ...]: + """The spellings ``resolve_callable`` accepts for a Go callable, beyond its full signature. + + Spec §7 requires ``DB.Query``, ``(*DB).Query``, ``sql.DB.Query`` and ``database/sql.DB.Query`` + all to name one method. A Go signature is ``..`` + (``example.com/dataflow/httpsql.sqlRepo.Find``), so: + + * the full signature and any **dot**-separated suffix of it are already handled by + :func:`~cldk.analysis.commons.resolve.segment_match` — that covers ``Find`` and + ``sqlRepo.Find``; + * the **slash** boundary is not, exactly as for a global's module half (:func:`go_value_names`), + so ``httpsql.sqlRepo.Find`` needs the package-qualified spelling as an explicit second name; + * Go's pointer-receiver syntax ``(*T).M`` is a spelling the analyzer never emits, so it is + added as a third. + + Returned as :attr:`~cldk.analysis.commons.resolve.CallableCandidate.match_names`, whose + contract is that a candidate matched by *any* of its names counts once — so these widen what + resolves without multiplying ambiguities. + """ + names = [signature] + head, _, tail = signature.rpartition("/") + if head: + names.append(tail) # package-qualified: httpsql.sqlRepo.Find + # (*T).M for the last two dotted segments, where there are two to take. + parts = names[-1].split(".") + if len(parts) >= 2: + names.append(f"(*{parts[-2]}).{parts[-1]}") + return tuple(dict.fromkeys(names)) + + +def value_display(var: str) -> Tuple[str, str, "str | None"]: + """``(kind, leaf, defined_in)`` for a ``formal_in`` vertex's ``var``, in the caller's vocabulary. + + A thin pass-through to :func:`~cldk.analysis.commons.resolve.value_candidate`, named here so + both Go backends describe a port identically and neither reads the raw ``:…`` / + ``:…`` markers into a user-facing field. Go's markers are the analyzer's own + (``internal/sdg/vertices.go``) and are spelled exactly as Python's, which is why the shared + translation applies unchanged. + """ + v = value_candidate(var) + return v.kind, v.leaf, v.defined_in + + +def dedupe(items: Sequence[str]) -> List[str]: + """``items`` with duplicates dropped, first occurrence winning — order is part of the contract + wherever a resolver's candidate list feeds an error message.""" + return list(dict.fromkeys(items)) From e480919f11ba3ac13af18f0788d1b1c3fde65266 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 5 Nov 2026 11:30:02 -0500 Subject: [PATCH 03/15] feat(go): answer the L3/L4 surface from the local backend Lifts the -a 2 cap in _argv: cango has emitted L3/L4 since 0.2.0, and capping a level-4 request at 2 silently produced a shallow analysis whose empty dataflow was indistinguishable from a proved absence. The backend builds the cross-callable SDG index out of the five lists the payload already carries -- ddg/cdg/summary keyed by local body key, param_in /param_out already absolute -- which are the same five the Neo4j projector emits, so the answer is the graph's answer rather than an intraprocedural one presented as complete. resolve_callable's domain includes the library stand-ins: spec section 7's own example, database/sql.DB.Query, is one, and a stand-in's formal_in ports are real L4 vertices a flow passes through. Measured on testdata/dataflow at -a 4: the four receiver spellings all resolve; a method's receiver resolves by `of` as formal_in:0; the two packages that both export lastQuery are ambiguous by leaf and split by the package-qualified spelling; and the gated HTTP->SQL flow answers True for flows_to_call and flows_to_argument, with paths_between returning the id -> n -> query -> DB.Query.query witness. Refs #436 --- cldk/analysis/go/codeanalyzer/codeanalyzer.py | 648 +++++++++++++++++- cldk/analysis/go/ids.py | 12 + cldk/core.py | 6 +- 3 files changed, 650 insertions(+), 16 deletions(-) diff --git a/cldk/analysis/go/codeanalyzer/codeanalyzer.py b/cldk/analysis/go/codeanalyzer/codeanalyzer.py index bb85ab0e..daca5f94 100644 --- a/cldk/analysis/go/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/go/codeanalyzer/codeanalyzer.py @@ -28,28 +28,64 @@ import os import shlex import subprocess +from functools import cached_property, partial from pathlib import Path from subprocess import CompletedProcess -from typing import Dict, List, Union +from typing import Dict, FrozenSet, Iterator, List, Mapping, Sequence, Tuple, Union import networkx as nx from cldk.analysis.commons.identity import resolve_identity, validate_identity, cache_matches from cldk.analysis.commons.backend_config import cache_subdir +from cldk.analysis.commons.bounds import ( + DEFAULT_DEPTH, + DEFAULT_MAX_NODES, + DEFAULT_MAX_PATHS, + DEFAULT_PAGE_SIZE, + check_depth, + check_distinct_endpoints, + check_max_nodes, + check_max_paths, + check_page_size, + edge_page, +) +from cldk.analysis.commons.graphs import call_reaches, cone_sinks, flow_path, shortest_walks, slice_resolved, under_callable from cldk.analysis.commons.levels import analyzer_level -from cldk.analysis.go.backend import ARTIFACT_LAYER_UNAVAILABLE, GoAnalysisBackend +from cldk.analysis.commons.resolve import ( + CallableCandidate, + body_node_kind, + resolve_callable_signature, + resolve_value_name, + resolve_within, +) +from cldk.analysis.commons.results import Diagnostic, EdgePage, FlowPath, FlowPaths, Slice, SliceNode +from cldk.analysis.go.backend import ( + ARTIFACT_LAYER_UNAVAILABLE, + CDG_ORDER, + CFG_ORDER, + DDG_ORDER, + PORTS_DISCONNECTED, + VIA, + GoAnalysisBackend, +) +from cldk.analysis.go.ids import body_node_id, callable_match_names, external_signature, go_value_names from cldk.models.go import ( GoAnalysis, GoApplication, GoCallable, GoCallGraphEdge, + GoCdgEdge, + GoCfgEdge, + GoDdgEdge, + GoDegraded, + GoExternalSymbol, GoField, GoModule, GoParam, GoType, ) from cldk.models.python import PyArtifact, PyConfigKey, PyConfigRead, PyConfigUseEdge, PyDependency -from cldk.utils.exceptions.exceptions import CodeanalyzerExecutionException +from cldk.utils.exceptions.exceptions import CodeanalyzerExecutionException, CodeanalyzerUsageException class GoCodeanalyzer(GoAnalysisBackend): @@ -85,8 +121,9 @@ def __init__( self.target_files = target_files # analyzer_level maps the AnalysisLevel enum/name/value to its integer, raising on an # unknown level rather than silently defaulting to 1 (the bug a hand-rolled dict caused). - # cango has no L3/L4, so _argv caps the result at 2. - self.analysis: GoAnalysis = self._init_codeanalyzer(analyzer_level(analysis_level)) + # _argv caps the result at _MAX_LEVEL, which is 4 since codeanalyzer-go 0.2.0. + self._level: int = analyzer_level(analysis_level) + self.analysis: GoAnalysis = self._init_codeanalyzer(self._level) validate_identity(self.analysis, self.identity) self.application: GoApplication = self.analysis.application self.application_name: str = self.identity.application_name @@ -115,11 +152,23 @@ def _get_codeanalyzer_exec(self) -> List[str]: "Install it with `pip install codeanalyzer-go`, or set $CODEANALYZER_GO_BIN." ) from e + #: The analysis level cango can be asked for. 4 since codeanalyzer-go 0.2.0, which added the + #: L3 intraprocedural graphs and the L4 interprocedural SDG (``-a 3`` / ``-a 4``, PR + #: codeanalyzer-go#36). Before it, every request was capped at 2 because nothing deeper existed + #: to ask for; leaving that cap in place is what made a level-4 request silently yield a + #: level-2 analysis, and then an empty dataflow answer indistinguishable from a proved absence. + _MAX_LEVEL = 4 + + #: The level at which ``cfg``/``cdg``/``ddg`` first exist, so :meth:`_require_dataflow` names it + #: rather than hard-coding a bare 3 at each call site. + _DATAFLOW_LEVEL = 3 + def _argv(self, analysis_level: int, output_dir: Path | None) -> List[str]: - """The cango command line: ``-i --app-name -a <1|2> + """The cango command line: ``-i --app-name --namespace -a <1..4> [-o --cache-dir ] --skip-tests [--eager] [-t ]...``. - . The level is capped at 2 — cango produces no L3/L4. + The level is capped at :attr:`_MAX_LEVEL`, which is what the binary accepts, so a caller + asking for more gets the deepest analysis available rather than a ``cango`` usage error. """ project = Path(self.project_dir) args = self._get_codeanalyzer_exec() + [ @@ -128,7 +177,7 @@ def _argv(self, analysis_level: int, output_dir: Path | None) -> List[str]: f"--app-name={self.identity.application_name}", f"--namespace={self.identity.namespace}", "-a", - str(min(analysis_level, 2)), + str(min(analysis_level, self._MAX_LEVEL)), ] if output_dir is not None: args += ["-o", str(output_dir), "--cache-dir", str(output_dir)] @@ -156,8 +205,9 @@ def _init_codeanalyzer(self, analysis_level: int) -> GoAnalysis: With no cache directory the output is read from the subprocess stdout pipe; with one, the analysis.json is persisted under ``/go`` and reused unless ``eager_analysis``, ``target_files``, or a cache **shallower than the requested level** force a re-run. A cache - at or above the requested level is reused as-is: the schema is additive (``L1 ⊆ L2``), so a - deeper cache already contains everything a shallower request needs. + at or above the requested level is reused as-is: the schema is additive + (``L1 ⊆ L2 ⊆ L3 ⊆ L4``), so a deeper cache already contains everything a shallower request + needs. """ output_dir = cache_subdir(self.cache_dir, self.project_dir, "go", identity=self.identity) if output_dir is None: @@ -170,9 +220,9 @@ def _init_codeanalyzer(self, analysis_level: int) -> GoAnalysis: output_dir.mkdir(parents=True, exist_ok=True) analysis_json_file = output_dir / "analysis.json" - # cango caps at 2, so a request above 2 is satisfied by a level-2 cache (never re-runs for a - # level it cannot produce). Compare the requested level, capped, against the cached level. - requested = min(analysis_level, 2) + # A request above what cango can produce is satisfied by a cache at the cap (never re-runs + # for a level it cannot produce). Compare the requested level, capped, against the cached. + requested = min(analysis_level, self._MAX_LEVEL) cached_level = self._cached_level(analysis_json_file) if analysis_json_file.exists() else None stale = cached_level is None or cached_level < requested or not cache_matches(analysis_json_file, self.identity) needs_run = self.eager_analysis or bool(self.target_files) or stale @@ -312,7 +362,577 @@ def get_source(self, node_id: str) -> str: self._module_bytes[mod.id] = encoded return encoded[b0:b1].decode("utf-8") - # -----[ artifact / dependency / config layer — not emitted by a local L1/L2 analysis ]----- + # ===================================================================================== + # L3/L4 DATAFLOW. See :class:`~cldk.analysis.go.backend.GoAnalysisBackend` for the contract and + # for the four Go-specific traps; the id/name grammar those traps need is in + # :mod:`cldk.analysis.go.ids` and is shared with the Neo4j backend so the two cannot drift. + # + # THIS BACKEND ANSWERS INTERPROCEDURALLY, for the reason the Python local backend does: a + # level-4 run carries the whole SDG in the payload, just split across five lists — + # + # GoCallable.ddg / .cdg endpoints are callable-LOCAL body keys -> joined by body_node_id + # GoCallable.summary a callee's pass-through at a call site, also local + # GoApplication.param_in actual_in -> formal_in, endpoints ALREADY absolute ids + # GoApplication.param_out formal_out -> actual_out, likewise + # + # — which are exactly the five the Neo4j projector emits as GO_DDG / GO_CDG / GO_SUMMARY / + # GO_PARAM_IN / GO_PARAM_OUT. So the cross-callable index this backend lacks it can BUILD out of + # the data the graph is projected from, and the answer it gives is the graph's answer rather + # than a narrower intraprocedural one presented as complete. + # + # The honest cost: building it walks every callable once and is cached for the backend's life. + # For the projects a local analysis is run on that is the cheaper trade; at Kubernetes scale + # (1.71M SDG nodes, 5.78M dependence edges) it is not, and the answer there is the Neo4j + # backend, which pushes the same traversal into Cypher. That asymmetry is why this backend's + # docstring warns about ``-a 4`` payload size. + # ===================================================================================== + + def _require_dataflow(self) -> None: + """Refuse, naming both levels, when this analysis was built below the dataflow pass. + + One guard for the per-callable graphs and the slices: they come from the same analyzer pass + and go dark together. It **raises** rather than returning empty because at a shallower level + an empty answer would mean "not analysed" while looking exactly like "no dependence" (D7). + """ + if self._level < self._DATAFLOW_LEVEL: + raise CodeanalyzerUsageException( + f"control and data flow need analysis_level 3 or deeper (cango -a 3 for the " + f"per-callable cfg/cdg/ddg, -a 4 for the interprocedural SDG); this analysis was " + f"built at level {self._level}, where the analyzer emits no cfg/cdg/ddg at all. " + "Returning an empty result would be indistinguishable from a callable that has no " + "dependence, so this raises instead. Rebuild with " + "CLDK.go(..., analysis_level='system_dependency_graph')." + ) + + @property + def _ports_carry_dependence(self) -> bool: + """Whether any ``formal_in`` vertex of this application has an outgoing SDG edge. + + Asked of the data (see the ABC), and answered off the adjacency this backend already builds, + so the probe costs nothing a slice would not have paid anyway. ``param_in`` alone is not the + question: an edge *into* a port says the caller side was emitted, while what a slice needs + is an edge *leaving* one — which is what carries the flow onward through the callee. + """ + if self._level < self._DATAFLOW_LEVEL: + return False + forward = self._sdg[0]["forward"] + return any(forward.get(ref) for ref in self._port_refs) + + def _require_connected_ports(self, accessor: str) -> None: + """Refuse ``accessor`` when this application's ports carry no dependence (see the ABC).""" + self._require_dataflow() + if not self._ports_carry_dependence: + raise CodeanalyzerExecutionException(PORTS_DISCONNECTED.format(accessor=accessor, app=self.application_name)) + + # -----[ the SDG index ]----- + def _iter_callables(self) -> Iterator[Tuple[GoCallable, str | None, str]]: + """Every callable with its owning type id and module path — the resolution domain. + + Nested closures included (``GoCallable.callables``), because the analyzer gives each its own + id, body and dataflow, and a flow through a Go closure is an ordinary flow. The owning type + id is ``None`` for a package-level function and for a closure nested in one. + """ + for path, mod in self.application.symbol_table.items(): + for fn in mod.functions.values(): + yield from self._descend(fn, None, path) + for ty in mod.types.values(): + for m in ty.callables.values(): + yield from self._descend(m, ty.id, path) + + def _descend(self, c: GoCallable, type_id: str | None, path: str) -> Iterator[Tuple[GoCallable, str | None, str]]: + yield c, type_id, path + for inner in c.callables.values(): + yield from self._descend(inner, type_id, path) + + @cached_property + def _dataflow_owners(self) -> Dict[str, Tuple[GoCallable, str | None, str]]: + """``{callable id: (callable, owning type id, module path)}``, stand-ins included. + + The stand-ins are in here because a library callable's ``formal_in``/``formal_out`` vertices + are real nodes on real paths at L4 — a flow through ``database/sql.DB.Query`` passes through + them — so a walk that could not describe them would report a path with holes in it. They + carry no module path (``""``), which is what tells :meth:`_slice_node` to describe them as + externals rather than inventing a file. + """ + owners: Dict[str, Tuple[GoCallable, str | None, str]] = {c.id: (c, tid, path) for c, tid, path in self._iter_callables()} + for sym in self.application.external_symbols.values(): + owners.setdefault(sym.id, (sym, None, "")) + return owners + + @cached_property + def _sdg(self) -> Tuple[Dict[str, Dict[str, Dict[str, list]]], Dict[str, Tuple[object, str, str]]]: + """``(adjacency, node index)`` over the whole application's SDG, built once and cached. + + ``adjacency`` is ``{"forward": {src: {dst: [label]}}, "backward": {dst: {src: [label]}}}`` — + both directions, because a backward slice is not derivable from a forward index without + inverting it, and inverting per call is the same work repeated. + + A ``label`` is ``(relationship type, var, prov)``: what a path hop has to report, and what + the graph carries on the corresponding relationship. It is a **list** per ``(src, dst)`` + pair because parallel edges are ordinary — one statement feeding one argument on several + variables is several distinct paths — and collapsing them would merge several pieces of + evidence into one. + + ``node index`` maps an absolute body-node id to ``(owning callable, local body key, module + path)``, which is everything :meth:`_slice_node` needs to describe a reached node without a + second walk. + """ + forward: Dict[str, Dict[str, list]] = {} + backward: Dict[str, Dict[str, list]] = {} + nodes: Dict[str, Tuple[object, str, str]] = {} + + def link(src: str, dst: str, label: tuple) -> None: + forward.setdefault(src, {}).setdefault(dst, []).append(label) + backward.setdefault(dst, {}).setdefault(src, []).append(label) + + for cid, (c, _tid, path) in self._dataflow_owners.items(): + for key in c.body or {}: + nodes[body_node_id(cid, key)] = (c, key, path) + # The relationship name each list is projected as, so a hop reports the same ``via`` + # here as it does over the graph — ``GoAnalysisBackend.VIA`` is the single translation + # table and neither backend has its own. + for rel, edges in (("GO_DDG", getattr(c, "ddg", None)), ("GO_CDG", getattr(c, "cdg", None)), ("GO_SUMMARY", getattr(c, "summary", None))): + for e in edges or []: + link(body_node_id(cid, e.src), body_node_id(cid, e.dst), (rel, getattr(e, "var", None), tuple(getattr(e, "prov", None) or ()))) + # These two carry absolute ids already (the analyzer resolved them through its identity + # map), so they are used as-is — joining them again would mint ids that name nothing. + for rel, edges in (("GO_PARAM_IN", self.application.param_in), ("GO_PARAM_OUT", self.application.param_out)): + for e in edges or []: + link(e.src, e.dst, (rel, e.var, ())) + return {"forward": forward, "backward": backward}, nodes + + @cached_property + def _port_refs(self) -> Tuple[str, ...]: + """Every ``formal_in`` vertex id in the application — the domain + :attr:`_ports_carry_dependence` probes and :meth:`flows_to_call` targets.""" + return tuple(body_node_id(cid, key) for cid, (c, _t, _p) in self._dataflow_owners.items() for key, n in (c.body or {}).items() if n.kind == "formal_in") + + def _slice_node(self, ref: str) -> SliceNode | None: + """One reached body node as a :class:`SliceNode`, or ``None`` for an id this app does not own. + + The Neo4j twin must describe a node identically; the two share + :func:`~cldk.analysis.commons.resolve.body_node_kind` (which decides the caller-facing kind + and name) and differ only in where the fields are read from. A parameter-passing vertex has + no span of its own, so the *callable's* first line stands in. + """ + entry = self._sdg[1].get(ref) + if entry is None: + return None + c, key, path = entry + node = (c.body or {})[key] + kind, name, defined_in = body_node_kind(node.kind, node.of) + line = node.span.start[0] if node.span else (c.span.start[0] if getattr(c, "span", None) else 0) + return SliceNode(file=path, line=line, callable=c.signature or c.id, kind=kind, name=name, defined_in=defined_in, source=None, ref=ref) + + # -----[ addressing ]----- + @cached_property + def _candidates(self) -> Tuple[List[CallableCandidate], Dict[str, Tuple[object, str | None, str]], FrozenSet[str]]: + """``(candidates, {signature: owner}, colliding signatures)`` for :meth:`resolve_callable`. + + **The library stand-ins are in the domain**, and they have to be: spec §7's own worked + example is ``database/sql.DB.Query``, which is a stand-in, and ``flows_to_call(…, + "DB.Query")`` is the question the HTTP→SQL fixture exists to ask. They resolve to the + ``can://…/@external/…`` id, which is a real vertex of the L4 graph with real ``formal_in`` + ports — so a flow into one is a flow a slice can follow, not a boundary. + + A stand-in's reported signature is :func:`~cldk.analysis.go.ids.external_signature` and its + ``path`` is ``""``: there is no file, and inventing one would put a fiction in a field a + caller reads. An empty ``module_names`` (not ``None``) is deliberate — it means "this + candidate supplies no dotted module name", so ``in_module=`` never matches a stand-in rather + than matching one through a path derivation that would be meaningless here (the same + distinction Java's J-2 records). + + Two callables sharing a signature collapse into one entry, and the resolver would then hand + back whichever won with no signal — so the collisions are recorded and a name resolving onto + one raises. Checked on the *answer* rather than eagerly, so a duplicate elsewhere does not + break every unrelated resolution. + """ + found: Dict[str, Tuple[object, str | None, str]] = {} + collisions: set = set() + for c, tid, path in self._iter_callables(): + if c.signature in found: + collisions.add(c.signature) + found[c.signature] = (c, tid, path) + cands = [ + CallableCandidate(sig, tid, path, match_names=callable_match_names(sig), module_names=(pkg,) if (pkg := self._package_of(path)) else ()) + for sig, (_c, tid, path) in found.items() + ] + for sym in self.application.external_symbols.values(): + sig = external_signature(sym.module, sym.signature, sym.name) + if sig in found: + collisions.add(sig) + found[sig] = (sym, None, "") + cands.append(CallableCandidate(sig, None, "", match_names=callable_match_names(sig), module_names=())) + return cands, found, frozenset(collisions) + + def _package_of(self, path: str) -> str | None: + """The ``package`` clause of the module at ``path`` — what ``in_module=`` matches a Go + callable's module against, besides the path itself. + + Supplied explicitly rather than derived: the shared + :func:`~cldk.analysis.commons.keys.module_dotted` strips a ``.py`` suffix and would leave + ``httpsql/httpsql.go`` as a dotted ``httpsql.httpsql.go``, which names nothing (the same + trap Java's J-2 records). A Go module's readable module name is its package clause. + """ + mod = self.application.symbol_table.get(path) + return mod.package if mod else None + + def resolve_callable(self, name: str, *, in_class: str | None = None, in_module: str | None = None) -> SliceNode: + """The one callable ``name`` names (see :meth:`GoAnalysisBackend.resolve_callable`). + + Nothing is pre-filtered: this backend has the whole list in memory, so there is no round + trip to save, and handing the shared policy the unfiltered domain is the strongest form of + "both backends resolve over the same set". The Go receiver spellings come from + :func:`~cldk.analysis.go.ids.callable_match_names`, which is shared with the Neo4j backend. + """ + cands, found, collisions = self._candidates + sig = resolve_callable_signature(name, cands, in_class=in_class, in_module=in_module, by_full_name="more of the import path") + if sig in collisions: + raise ValueError(f"{sig!r} is carried by more than one analysed callable; neither can be addressed unambiguously") + c, _tid, path = found[sig] + if not c.id: + raise KeyError(f"{sig!r} carries no usable address (id={c.id!r})") + # A stand-in has no file and no span, so it is described as an external rather than given a + # fabricated position — the same decision :meth:`_callable_node` makes for a path's vertices. + span = getattr(c, "span", None) + return SliceNode( + file=path, + line=span.start[0] if span else 0, + callable=sig, + kind="callable" if path else "external", + name=sig.rpartition(".")[2], + source=None, + ref=c.id, + ) + + def resolve_value(self, name: str, *, within: str) -> SliceNode: + """The one value of ``within`` that ``name`` names (see + :meth:`GoAnalysisBackend.resolve_value`). + + The domain is the resolved callable's ``formal_in`` vertices, read by their own ``of`` — + never by ordinal into ``parameters``, because a method's ``formal_in:0`` is the receiver + (trap 2). Each is given every spelling a Go caller may write it as, which is where the + package-qualified form of a global comes from (trap 3, + :func:`~cldk.analysis.go.ids.go_value_names`). + + A **list** of ``(key, name)`` pairs rather than a dict keyed by name: two values resolving + to the same name are a genuine ambiguity the policy must see and raise on, and a dict would + silently keep the last. + """ + owner = resolve_within(self.resolve_callable, within) + c = self._dataflow_owners[owner.ref][0] + entries = [(key, nm) for key, node in (c.body or {}).items() if node.kind == "formal_in" and node.of for nm in go_value_names(node.of)] + chosen = resolve_value_name(name, [nm for _k, nm in entries], within=owner.callable) + key = next(k for k, nm in entries if nm == chosen) + node = (c.body or {})[key] + kind, leaf, defined_in = body_node_kind(node.kind, node.of) + return SliceNode(file=owner.file, line=owner.line, callable=owner.callable, kind=kind, name=leaf, defined_in=defined_in, source=None, ref=body_node_id(c.id, key)) + + # -----[ per-callable graphs ]----- + def _graphs_of(self, name: str, in_class: str | None, page_size: int) -> GoCallable: + """The callable ``name`` resolves to, once this backend is deep enough to have dataflow. + + ``page_size`` is validated **first**, before the level guard and before resolution, so a + malformed argument is a ``ValueError`` before anything else — the order the Neo4j backend + applies too, so the two cannot answer the same bad call with different exceptions. + """ + check_page_size(page_size) + self._require_dataflow() + node = self.resolve_callable(name, in_class=in_class) + return self._dataflow_owners[node.ref][0] + + def get_cfg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoCfgEdge]: + """One page of control flow within one callable (see :meth:`GoAnalysisBackend.get_cfg`). + + ``GoCallable.cfg`` keys its endpoints by the *local* body key (``"10:25"``, ``"@entry"``); + :func:`~cldk.analysis.go.ids.body_node_id` joins them to the callable id to give the same + absolute spelling the graph merges ``:GoBodyNode`` on, so an endpoint from this backend is + the one the Neo4j backend would return. The join happens *before* the sort, because the + order is over the ids a caller sees, not over the local keys. + """ + c = self._graphs_of(callable, in_class, page_size) + edges = [GoCfgEdge(src=body_node_id(c.id, e.src), dst=body_node_id(c.id, e.dst), kind=e.kind) for e in c.cfg or []] + return edge_page(GoCfgEdge, c.signature, edges, CFG_ORDER, page_size, cursor) + + def get_cdg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoCdgEdge]: + """One page of control dependence within one callable (see :meth:`GoAnalysisBackend.get_cdg`).""" + c = self._graphs_of(callable, in_class, page_size) + edges = [GoCdgEdge(src=body_node_id(c.id, e.src), dst=body_node_id(c.id, e.dst)) for e in c.cdg or []] + return edge_page(GoCdgEdge, c.signature, edges, CDG_ORDER, page_size, cursor) + + def get_ddg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoDdgEdge]: + """One page of data dependence within one callable (see :meth:`GoAnalysisBackend.get_ddg`).""" + c = self._graphs_of(callable, in_class, page_size) + edges = [GoDdgEdge(src=body_node_id(c.id, e.src), dst=body_node_id(c.id, e.dst), var=e.var, prov=list(e.prov or [])) for e in c.ddg or []] + return edge_page(GoDdgEdge, c.signature, edges, DDG_ORDER, page_size, cursor) + + # -----[ slicing and reachability ]----- + def _reach(self, ref: str, direction: str, depth: int | None) -> set: + """The node ids reachable from ``ref`` in at most ``depth`` hops. + + Level-by-level rather than a plain stack, because ``depth`` is a hop budget and a + depth-first walk cannot count hops without revisiting. Shared by the slices and by the two + flow predicates, so "reachable" means one thing on this backend. + """ + edges = self._sdg[0][direction] + seen, frontier, hops = {ref}, [ref], 0 + while frontier and (depth is None or hops < depth): + nxt = [d for src in frontier for d in edges.get(src, ()) if d not in seen] + seen.update(nxt) + frontier = nxt + hops += 1 + return seen + + def _slice_from(self, root: SliceNode, direction: str, depth: int | None, max_nodes: int) -> Slice: + """:meth:`_reach`'s closure from ``root``, described and capped as the graph's is. + + The whole closure is computed and *then* cut: ``total`` has to be the size of the whole + slice for the cap to be reportable, and there is nothing cheaper to compute it from — the + same reason the Cypher counts before it pages. + """ + found = [n for ref in sorted(self._reach(root.ref, direction, depth)) if (n := self._slice_node(ref)) is not None] + return Slice(nodes=found[:max_nodes], roots=[root], resolved=slice_resolved([root]), total=len(found)) + + def slice_backward(self, src: str, *, within: str, depth: int | None = DEFAULT_DEPTH, max_nodes: int = DEFAULT_MAX_NODES) -> Slice: + """What affects this value (see :meth:`GoAnalysisBackend.slice_backward`).""" + check_depth(depth) + check_max_nodes(max_nodes) + self._require_connected_ports("slice_backward") + return self._slice_from(self.resolve_value(src, within=within), "backward", depth, max_nodes) + + def slice_forward(self, src: str, *, within: str, depth: int | None = DEFAULT_DEPTH, max_nodes: int = DEFAULT_MAX_NODES) -> Slice: + """What this value affects (see :meth:`GoAnalysisBackend.slice_forward`).""" + check_depth(depth) + check_max_nodes(max_nodes) + self._require_connected_ports("slice_forward") + return self._slice_from(self.resolve_value(src, within=within), "forward", depth, max_nodes) + + def reaches(self, src: str, dst: str, *, depth: int | None = None) -> bool: + """Is there a call path (see :meth:`GoAnalysisBackend.reaches`)? + + Over :meth:`get_call_graph`, the same edge set :meth:`get_callers` / :meth:`get_callees` + read, so the boolean cannot disagree with the neighbours a caller can enumerate. The Go + call graph is keyed by callable **id**, not by signature, so the resolved ``ref`` is the + vertex — which is also why this needs no symbol-table lookup afterwards. + """ + check_depth(depth) + a = self.resolve_callable(src).ref + b = self.resolve_callable(dst).ref + # ``call_reaches`` owns the zero-hop rule: no path is vacuous, so ``reaches(x, x)`` is the + # cycle question rather than trivially True. + return call_reaches(self.get_call_graph(), a, b, depth) + + def backward_cone(self, sinks: Sequence[str], *, depth: int | None = DEFAULT_DEPTH, max_nodes: int = DEFAULT_MAX_NODES) -> Slice: + """Every callable that can reach these sinks (see :meth:`GoAnalysisBackend.backward_cone`).""" + check_depth(depth) + check_max_nodes(max_nodes) + roots = cone_sinks(self.resolve_callable, sinks) + graph = self.get_call_graph() + reached: set = set() + for root in roots: + reached.add(root.ref) + if root.ref in graph: + # ``ego_graph`` follows successors, so a backward question needs the reversed view. + back = graph.reverse(copy=False) + reached |= nx.ancestors(graph, root.ref) if depth is None else set(nx.ego_graph(back, root.ref, radius=depth, undirected=False).nodes) + found = sorted((n for n in (self._callable_node(ref) for ref in reached) if n is not None), key=lambda n: n.ref) + return Slice(nodes=found[:max_nodes], roots=roots, resolved=slice_resolved(roots), total=len(found)) + + def _callable_node(self, callable_id: str) -> SliceNode | None: + """The callable ``callable_id`` names, as a :class:`SliceNode`. + + A call-graph vertex the symbol table does not declare is a library stand-in; it is described + from :meth:`get_external_symbols` rather than by parsing its id, which is the same decision + the Neo4j backend makes off the ``:GoExternal`` node's properties. + """ + entry = self._dataflow_owners.get(callable_id) + if entry is not None: + c, _tid, path = entry + if path: + return SliceNode(file=path, line=c.span.start[0] if c.span else 0, callable=c.signature, kind="callable", name=c.signature.rpartition(".")[2], source=None, ref=callable_id) + sym = self.application.external_symbols.get(callable_id) + if sym is not None: + return SliceNode(file="", line=0, callable=f"{sym.module}.{sym.signature or sym.name}", kind="external", name=sym.name, source=None, ref=callable_id) + return None + + # -----[ paths and the flow predicates ]----- + #: Up to ``limit`` shortest walks over a ``{src: {dst: [label]}}`` adjacency, in the documented + #: hop order. The ``via`` table is the one parameter: the algorithm is a graph walk over strings + #: and knows no language. + _shortest_walks = staticmethod(partial(shortest_walks, via=VIA)) + + def _describe_walks(self, walks: list, a: SliceNode) -> List[FlowPath]: + """``walks`` as :class:`FlowPath` objects, with every node described once.""" + described = {ref: n for walk in walks for ref, _ in walk if (n := self._slice_node(ref)) is not None} + described[a.ref] = a + return [flow_path([described[a.ref]] + [described[ref] for ref, _ in walk if ref in described], [label for _, label in walk], via=VIA) for walk in walks] + + def paths_between(self, src: str, dst: str, *, src_within: str, dst_within: str, depth: int | None = None, max_paths: int = DEFAULT_MAX_PATHS) -> FlowPaths: + """How a value reaches another value (see :meth:`GoAnalysisBackend.paths_between`).""" + check_depth(depth) + check_max_paths(max_paths) + self._require_connected_ports("paths_between") + a = self.resolve_value(src, within=src_within) + b = self.resolve_value(dst, within=dst_within) + check_distinct_endpoints(a, b) + walks = self._shortest_walks(self._sdg[0]["forward"], a.ref, b.ref, depth, max_paths + 1) + return FlowPaths(paths=self._describe_walks(walks[:max_paths], a), complete=len(walks) <= max_paths) + + def _callee_ports(self, callable_id: str) -> List[str]: + """The ids of every value *entering* this callable — its ``formal_in`` vertices. + + Read off ``body`` by ``kind``, not by ordinal into ``parameters``: on a method the receiver + is one of these and is absent from ``parameters`` (trap 2), and a closure's captures are + formals with no parameter at all. + """ + entry = self._dataflow_owners.get(callable_id) + if entry is None: + return [] + c = entry[0] + return [body_node_id(callable_id, key) for key, node in (c.body or {}).items() if node.kind == "formal_in"] + + def flows_to_call(self, src: str, callee: str, *, within: str, depth: int | None = None) -> bool: + """Does this value reach any value entering a call to ``callee`` + (see :meth:`GoAnalysisBackend.flows_to_call`)?""" + check_depth(depth) + self._require_connected_ports("flows_to_call") + root = self.resolve_value(src, within=within) + targets = self._callee_ports(self.resolve_callable(callee).ref) + return bool(targets) and not self._reach(root.ref, "forward", depth).isdisjoint(set(targets) - {root.ref}) + + def flows_to_argument(self, src: str, callee: str, arg: str, *, within: str, depth: int | None = None) -> bool: + """Does this value reach ``callee``'s ``arg`` + (see :meth:`GoAnalysisBackend.flows_to_argument`)?""" + check_depth(depth) + self._require_connected_ports("flows_to_argument") + root = self.resolve_value(src, within=within) + target = self.resolve_value(arg, within=callee).ref + return target != root.ref and target in self._reach(root.ref, "forward", depth) + + # -----[ taint ]----- + def _taint_walk( + self, + srcs: Sequence[SliceNode], + dsts: Sequence[SliceNode], + *, + cuts: List[Dict[str, str]], + cut_callables: List[str], + depth: int | None, + max_paths: int, + ) -> Tuple[List[Tuple[str, str, FlowPath]], Mapping[Tuple[str, str], List[Diagnostic]]]: + """The sanitized shortest walks, in process (see :meth:`GoAnalysisBackend._taint_walk`). + + ``_require_dataflow`` first, so a *direct* call to this hook is diagnosed by level rather + than by an empty walk. Through :meth:`taint` it never fires: ``_require_connected_ports`` + runs there and subsumes it. + + One :func:`~cldk.analysis.commons.graphs.shortest_walks` call **per pair**, which is what + makes ``max_paths + 1`` a per-pair cap here the way ``collect(p)[0..$cap]`` is one over + Cypher — a single walk over the flattened lists would let one prolific pair starve the rest. + Pairs are deduplicated by resolved position first: two selectors naming one position are one + pair, and walking it twice would report each witness twice and make a cap of *m* yield *2m*. + + Both cuts are the walker's **predicates** rather than a filter over what it returns, which + is the property the whole design rests on: the breadth-first pass must measure the shortest + *satisfying* distance, or a sanitized short route hides a clean longer one and the pair comes + back refuted. + + The ledger comes back empty. Go's unresolved-dispatch signal is the application-scope + ``degraded[]``, which :meth:`taint` reads directly and files per pair — a per-frontier signal + this walk could file instead does not exist in the payload, and inventing one from, say, a + ``callee is None`` call node would void every refutation in every application that calls a + library, which is the over-reporting leg 4b's Ruling I rejects. + """ + self._require_dataflow() + forward = self._sdg[0]["forward"] + allow_node = (lambda nid: not under_callable(nid, cut_callables)) if cut_callables else None + allow_edge = (lambda frm, _rel, var: not any(var == c["var"] and under_callable(frm, (c["prefix"],)) for c in cuts)) if cuts else None + pairs: Dict[Tuple[str, str], SliceNode] = {} + for a in srcs: + for b in dsts: + pairs.setdefault((a.ref, b.ref), a) + rows: List[Tuple[str, str, FlowPath]] = [] + for (src_ref, dst_ref), a in pairs.items(): + walks = self._shortest_walks(forward, src_ref, dst_ref, depth, max_paths + 1, allow_edge=allow_edge, allow_node=allow_node) + rows.extend((src_ref, dst_ref, p) for p in self._describe_walks(walks, a)) + return rows, {} + + def _edge_vars_in(self, callable_id: str) -> FrozenSet[str]: + """The edge variables scoped to this callable (see :meth:`GoAnalysisBackend._edge_vars_in`). + + Off the adjacency this backend already caches, so a variable sanitizer costs a scan and no + second traversal. Edges *leaving* a node under ``callable_id`` — the same ``startNode`` + scoping the cut itself uses, so this validates exactly the domain the cut can match. + + "Under ``callable_id``" is :func:`~cldk.analysis.commons.graphs.under_callable` and not + ``startswith``: with a bare prefix test this domain would include every variable of a + sibling callable whose id merely starts with this one — in Go, every closure of a + same-named outer function — and a sanitizer naming one of those would be *accepted* here and + then sever nothing in ``allow_edge``, which is delimited. + """ + forward = self._sdg[0]["forward"] + return frozenset(var for src, outs in forward.items() if under_callable(src, (callable_id,)) for labels in outs.values() for _rel, var, _prov in labels if var) + + # -----[ the honesty signals + hydration ]----- + def get_external_symbols(self) -> Dict[str, GoExternalSymbol]: + """The library stand-ins this analysis synthesized (see + :meth:`GoAnalysisBackend.get_external_symbols`). + + Returned as-is rather than refused when empty: unlike Java's ``--external-calls``, Go emits + the stand-ins as part of ``-a 4`` with no opt-in, so an empty map at level 4 genuinely means + "this application calls nothing outside itself". + """ + return self.application.external_symbols + + def get_degraded(self) -> List[GoDegraded]: + """This analysis's degradation markers (see :meth:`GoAnalysisBackend.get_degraded`). + + Envelope scope, not application scope — ``degraded`` sits beside ``application`` on the + wire (``internal/schema/v2/schema.go``), which is why this reads ``self.analysis`` and not + ``self.application``. + """ + return self.analysis.degraded + + def describe(self, nodes: Sequence[object]) -> List[SliceNode]: + """Fill in ``source`` for these positions (see :meth:`GoAnalysisBackend.describe`). + + A body node's text is sliced out of the owning module by its span, so this fills in + statements and call sites. A vertex with **no** span — the synthetic ``@entry``/``@exit`` and + every ``formal_in``/``formal_out`` — gets ``None``: it is a dataflow position, not a region + of the file, and there is nothing to read on either backend. + """ + out: List[SliceNode] = [] + for n in nodes: + node = n if isinstance(n, SliceNode) else self._slice_node(str(n)) + if node is None: + continue + out.append(node.model_copy(update={"source": self._node_source(node.ref)})) + return out + + def _node_source(self, ref: str) -> str | None: + """The text of ``ref``, or ``None`` when it has no span (so nothing to read).""" + try: + return self.get_source(ref) + except (CodeanalyzerExecutionException, KeyError): + entry = self._sdg[1].get(ref) + if entry is None: + return None + c, key, _path = entry + node = (c.body or {})[key] + if node.span is None: + return None + mod = self._owning_module(c) + if mod is None: + return None + b0, b1 = node.span.bytes + return mod.source[b0:b1] if mod.source.isascii() else mod.source.encode("utf-8")[b0:b1].decode("utf-8") + + def _owning_module(self, c: object) -> GoModule | None: + """The module declaring ``c``, for slicing a body node's span out of its source.""" + entry = self._source_of.get(getattr(c, "id", "")) + return entry[0] if entry else None + + # -----[ artifact / dependency / config layer — not emitted by a local analysis ]----- def get_artifacts(self) -> Dict[str, PyArtifact]: raise CodeanalyzerExecutionException(ARTIFACT_LAYER_UNAVAILABLE.format(app=self.application_name)) diff --git a/cldk/analysis/go/ids.py b/cldk/analysis/go/ids.py index 82b57d44..aef694da 100644 --- a/cldk/analysis/go/ids.py +++ b/cldk/analysis/go/ids.py @@ -184,6 +184,18 @@ def callable_match_names(signature: str) -> Tuple[str, ...]: return tuple(dict.fromkeys(names)) +def external_signature(module: str, signature: str, name: str) -> str: + """The reported signature of a library stand-in: ``..``. + + Assembled from the stand-in's own ``module`` and ``signature`` properties rather than parsed + back out of its ``can://…/@external/…`` id, so it reads in the same vocabulary a project + callable's signature does: ``database/sql`` + ``DB.Query`` → ``database/sql.DB.Query``, which + is the spelling spec §7 requires :meth:`resolve_callable` to accept. ``signature`` is absent on + an older emission, in which case the bare ``name`` stands in. + """ + return f"{module}.{signature or name}" + + def value_display(var: str) -> Tuple[str, str, "str | None"]: """``(kind, leaf, defined_in)`` for a ``formal_in`` vertex's ``var``, in the caller's vocabulary. diff --git a/cldk/core.py b/cldk/core.py index 266c11e3..86744e94 100644 --- a/cldk/core.py +++ b/cldk/core.py @@ -242,8 +242,10 @@ def go( project_path: Path to the Go module directory (needs a ``go.mod``). Optional only when ``backend`` is a :class:`Neo4jConnectionConfig` (the graph is populated out of band over Bolt). When provided, the path is validated — it must exist and be a directory. - analysis_level: Analysis depth (see :class:`~cldk.analysis.AnalysisLevel`); ``cango`` - produces levels 1 (symbol table) and 2 (+ resolver call graph) only. + analysis_level: Analysis depth (see :class:`~cldk.analysis.AnalysisLevel`). ``cango`` + produces all four: 1 (symbol table), 2 (+ resolver call graph), 3 (+ the + per-callable ``cfg``/``cdg``/``ddg``) and 4 (+ the interprocedural SDG the + dataflow and ``taint`` accessors need). target_files: Restrict analysis to these files. eager: Force regeneration of cached analysis. backend: Backend configuration. Defaults to :class:`CodeAnalyzerConfig` (runs ``cango``); From 0459dd5edc301dba6e0d481b8484d97889a1ad08 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 5 Nov 2026 11:47:04 -0500 Subject: [PATCH 04/15] feat(go): answer the L3/L4 surface over Cypher, with live parity The Neo4j backend's half of the dataflow surface, plus the parity contract: both backends answer all 66 probed questions identically on the same application, miss paths and taint verdicts included. Four bugs the live graph found that no mocked test could, each fixed with the measurement in the comment: - the scope guard admitted only `/go/`, so every library stand-in id was refused as "outside the bound Go application scope" -- one cause behind 19 of 22 initial diffs. A stand-in is this application's node: the projector mints it under this prefix and it carries this app's ports. - :GoExternal is two things. 35 callable stand-ins carry a signature; 16 import ghosts (a package an unresolved import named) do not, and analysis.json carries only the first. get_external_symbols reported 51 against the local backend's 35. - _REACHES pinned every hop AND the endpoint to :GoCallable, so reaches(caller, "DB.Query") was False for a call that exists -- 69 of 247 GO_CALLS edges land on a stand-in. The interior stays pinned; the endpoint is now free, which cannot widen the interior because no stand-in has an outgoing call edge (0 of 247, in both the graph and the JSON). - reconstruct.body_node dropped `of`/`parent`: the graph spells them `var` and `call_node`, so every L4 vertex came back with of=None -- and `of` is what resolves a formal. The live suite's local fixture was also stale: it seeded /go rather than the identity-digest path cache_subdir has keyed since #434, so it wrote where nothing reads and the backend re-ran cango against /unused. Refs #436 --- cldk/analysis/go/neo4j/neo4j_backend.py | 728 +++++++++++++++++++++++- cldk/analysis/go/neo4j/reconstruct.py | 47 +- tests/analysis/go/test_go_neo4j_live.py | 284 ++++++++- 3 files changed, 1046 insertions(+), 13 deletions(-) diff --git a/cldk/analysis/go/neo4j/neo4j_backend.py b/cldk/analysis/go/neo4j/neo4j_backend.py index 6bf9590a..0c3ce94d 100644 --- a/cldk/analysis/go/neo4j/neo4j_backend.py +++ b/cldk/analysis/go/neo4j/neo4j_backend.py @@ -41,14 +41,55 @@ import json from functools import cached_property -from typing import Any, Dict, List +from typing import Any, Callable, Dict, FrozenSet, List, Mapping, Sequence, Tuple import networkx as nx +from cldk.analysis.commons.bounds import ( + DEFAULT_DEPTH, + DEFAULT_MAX_NODES, + DEFAULT_MAX_PATHS, + DEFAULT_PAGE_SIZE, + EdgeOrder, + check_depth, + check_distinct_endpoints, + check_max_nodes, + check_max_paths, + check_page_size, + cursor_params, + encode_cursor, + keyset_where, +) +from cldk.analysis.commons.graphs import cone_sinks, flow_path, sdg_path_query, sdg_taint_query, slice_resolved from cldk.analysis.commons.identity import index_module_rows, resolve_identity, scoped_module_keys, validate_identity -from cldk.analysis.go.backend import ARTIFACT_LAYER_UNAVAILABLE, GoAnalysisBackend +from cldk.analysis.commons.resolve import CallableCandidate, body_node_kind, resolve_callable_signature, resolve_value_name, resolve_within +from cldk.analysis.commons.results import Diagnostic, EdgePage, FlowPath, FlowPaths, Slice, SliceNode +from cldk.analysis.go.backend import ( + ARTIFACT_LAYER_UNAVAILABLE, + CDG_ORDER, + CFG_ORDER, + DDG_ORDER, + PORTS_DISCONNECTED, + SDG_REL_PATTERN, + VIA, + GoAnalysisBackend, +) +from cldk.analysis.go.ids import callable_match_names, external_signature, go_value_names from cldk.analysis.go.neo4j import reconstruct as R -from cldk.models.go import GoApplication, GoCallable, GoCallGraphEdge, GoField, GoModule, GoParam, GoType +from cldk.models.go import ( + GoApplication, + GoCallable, + GoCallGraphEdge, + GoCdgEdge, + GoCfgEdge, + GoDdgEdge, + GoDegraded, + GoExternalSymbol, + GoField, + GoModule, + GoParam, + GoType, +) from cldk.models.python import PyArtifact, PyConfigKey, PyConfigRead, PyConfigUseEdge, PyDependency from cldk.utils.exceptions.exceptions import CodeanalyzerExecutionException @@ -56,6 +97,67 @@ _REQUIRED_RELS = {"GO_HAS_MODULE", "GO_DECLARES", "GO_HAS_METHOD", "GO_CALLS"} +def _slice_node(row: Dict[str, Any], module_key: Callable[[str], str]) -> SliceNode: + """One row of a slice / path query as a :class:`SliceNode`, in the caller's vocabulary. + + The local twin is ``GoCodeanalyzer._slice_node``; the two share + :func:`~cldk.analysis.commons.resolve.body_node_kind`, so a vertex a caller addressed through + ``resolve_value`` as a ``global`` comes back from a slice labelled a ``global`` on either + backend, and they differ only in where the fields are read from. + + ``row["var"]`` is the graph's name for the JSON's ``of`` (see + :func:`~cldk.analysis.go.neo4j.reconstruct.body_node`), so it is what carries the value a + vertex stands for. + + A vertex that is a **library stand-in's** has no module — its id holds ``/@external/`` where a + project node holds a ``.go`` path — so ``file`` is left empty rather than passed through a + module-key derivation that would mint a path naming nothing. + """ + kind, name, defined_in = body_node_kind(row["kind"], row["var"]) + ref = row["ref"] + try: + file = module_key(ref) + except ValueError: + file = "" + return SliceNode( + file=file, + line=row["line"] if row["line"] is not None else row["c_line"], + callable=row["callable"] or ref, + kind=kind, + name=name, + defined_in=defined_in, + source=None, + ref=ref, + ) + + +def _call_neighbour(row: Dict[str, Any], module_key: Callable[[str], str]) -> SliceNode: + """One ``GO_CALLS`` neighbour as a :class:`SliceNode` — declared callable or library stand-in. + + Which it is, is read off the row rather than asked for in a second query: only a + ``:GoCallable`` carries ``signature`` *and* a module, and a stand-in carries ``module``/``name`` + (``internal/neo4j/schema.go``). A stand-in's reported name is + :func:`~cldk.analysis.go.ids.external_signature`, the same spelling the local backend reports + and the same one ``resolve_callable`` accepts, so a path's external hop reads identically on + both backends. + """ + try: + file = module_key(row["ref"]) + except ValueError: + file = "" + if file: + return SliceNode(file=file, line=row["line"] or 0, callable=row["signature"], kind="callable", name=row["name"], source=None, ref=row["ref"]) + return SliceNode( + file="", + line=0, + callable=external_signature(row["module"] or "", row["signature"], row["name"] or ""), + kind="external", + name=row["name"], + source=None, + ref=row["ref"], + ) + + class GoNeo4jBackend(GoAnalysisBackend): """Answer the Go query surface over a ``GO_``-prefixed Neo4j graph (see the module docstring).""" @@ -112,11 +214,33 @@ def _scope_prefix(self) -> str: """``can:////`` — the trailing slash keeps ``app`` from matching ``app-b``.""" return f"{self._app_id}/" + #: The segments that may follow ``can:////`` on a node id this application owns. + #: ``go/`` is project code — the language segment every declared node sits under — and + #: ``@external/`` is a **library stand-in**, which is equally this application's node: the + #: projector mints it under this app's prefix, it carries this app's ``formal_in`` ports, and a + #: flow *through* ``database/sql.DB.Query`` is a flow this graph holds. Admitting only ``go/`` + #: refused every stand-in id, which made ``taint``, ``paths_between`` and the two flow + #: predicates raise "outside the bound Go application scope" for a sink inside the application + #: — measured against the live graph, 19 of 22 parity diffs had this one cause. The foreign-owner + #: and foreign-language cases the guard exists for are unaffected: both differ before this + #: segment, in the ``can:////`` prefix itself. + _OWNED_SEGMENTS = ("go/", "@external/") + + def _in_scope(self, node_id: str) -> bool: + """Does ``node_id`` name a node *this* application owns (see :data:`_OWNED_SEGMENTS`)?""" + return any(node_id.startswith(self._scope_prefix + seg) for seg in self._OWNED_SEGMENTS) + def _run(self, query: str, **params: Any) -> List[Dict[str, Any]]: - """Run one read statement over a reused session; drop the session on failure.""" + """Run one read statement over a reused session; drop the session on failure. + + A ``can://`` id passed as a point-lookup parameter is checked against this application's + scope **before** the statement runs, so a foreign owner or a sibling language's node is + refused without a round trip rather than silently answering from another application's + subgraph. + """ for name in ("id", "mid", "tid"): value = params.get(name) - if isinstance(value, str) and value.startswith("can:") and not value.startswith(self._scope_prefix + "go/"): + if isinstance(value, str) and value.startswith("can:") and not self._in_scope(value): raise ValueError("Node URI is outside the bound Go application scope") if self._session_obj is None: self._session_obj = self._driver.session(database=self._database) @@ -340,6 +464,600 @@ def get_source(self, node_id: str) -> str: return source[b0:b1] return source.encode("utf-8")[b0:b1].decode("utf-8") + # ===================================================================================== + # L3/L4 DATAFLOW over Cypher (python-sdk#436). The contract and the four Go traps are on + # :class:`~cldk.analysis.go.backend.GoAnalysisBackend`; the id/name grammar is + # :mod:`cldk.analysis.go.ids`, shared with the local backend so the two cannot drift on it. + # + # NO LEVEL GATE HERE. ``cango --emit neo4j`` is always full depth (it takes no ``-a``), so an + # attached graph either carries the SDG or is not a dataflow projection at all — which is why + # ``_require_dataflow`` is the *local* backend's method. What this backend does share is + # :meth:`_require_connected_ports`, because a graph can carry the ports and still have nothing + # crossing into them, and that is a question only the data can answer. + # + # THE GRAPH RENAMES TWO BODY-NODE PROPERTIES: the JSON's ``of`` is ``b.var`` and its ``parent`` + # is ``b.call_node`` (``internal/neo4j/schema.go``). Every projection below reads ``b.var``, so + # the value a vertex stands for arrives under the name the shared helpers expect. + # + # A STAND-IN IS A FIRST-CLASS VERTEX. ``:GoExternal`` nodes carry ``GO_HAS_BODY_NODE`` body + # nodes at L4, so the traversals must not be labelled ``:GoCallable``-only on the interior or + # a flow *through* ``database/sql.DB.Query`` would be invisible — which is the very flow the + # httpsql fixture gates. ``:GoCallable|GoExternal`` is therefore deliberate everywhere below. + # ===================================================================================== + + #: Both a project callable and a library stand-in own body nodes at L4, so every containment hop + #: on this surface accepts either label (see the block comment above). + _OWNER = "GoCallable|GoExternal" + + def _require_connected_ports(self, accessor: str) -> None: + """Refuse ``accessor`` when this application's ports carry no dependence (see the ABC).""" + if not self._ports_carry_dependence: + raise CodeanalyzerExecutionException(PORTS_DISCONNECTED.format(accessor=accessor, app=self.application_name)) + + @cached_property + def _ports_carry_dependence(self) -> bool: + """Whether any ``formal_in`` vertex of this application has an outgoing SDG edge. + + One ``LIMIT 1`` existence query, cached: the probe reads the *data* rather than a version + string, exactly as the Java backend's does, so a graph re-emitted by a build that connects + the layers makes these accessors work with no change here. An edge *leaving* a port is the + question — an edge arriving at one only says the caller side was projected. + """ + rows = self._run( + f"MATCH (b:GoBodyNode {{kind:'formal_in'}})-[r:{SDG_REL_PATTERN}]->() WHERE b.id STARTS WITH $pre RETURN count(r) > 0 AS ok LIMIT 1", + pre=self._scope_prefix, + ) + return bool(rows and rows[0].get("ok")) + + # -----[ addressing ]----- + @cached_property + def _callable_candidates(self) -> List[CallableCandidate]: + """Every callable and stand-in this application declares, as resolution candidates. + + One scan, cached for the backend's life. The stand-ins are in the domain for the local + backend's reason: ``database/sql.DB.Query`` is spec §7's worked example and is one, and its + ``formal_in`` ports are real L4 vertices a flow passes through. + + A ``:GoCallable``'s ``in_module=`` name is its module's ``package`` clause, read from the + graph rather than derived from the path — the shared ``module_dotted`` derivation knows only + ``.py`` and would leave ``httpsql/httpsql.go`` as a dotted name that matches nothing. + """ + rows = self._run( + "MATCH (m:GoModule)-[:GO_HAS_MODULE|GO_DECLARES|GO_HAS_METHOD*0..2]->(c:GoCallable) WHERE c.id STARTS WITH $pre " + "OPTIONAL MATCH (t:GoType)-[:GO_HAS_METHOD]->(c) " + "RETURN DISTINCT c.signature AS signature, c.id AS ref, t.id AS type_id, m.package AS package ORDER BY c.signature", + pre=self._scope_prefix, + ) + cands = [ + CallableCandidate( + r["signature"], + r["type_id"], + self._path_of(r["ref"]), + match_names=callable_match_names(r["signature"]), + module_names=(r["package"],) if r["package"] else (), + ) + for r in rows + if r["signature"] + ] + for sig in self._external_signatures: + cands.append(CallableCandidate(sig, None, "", match_names=callable_match_names(sig), module_names=())) + return cands + + def _path_of(self, node_id: str) -> str: + """The module file key embedded in ``node_id``, or ``""`` for a stand-in (which has none).""" + try: + return self._module_key(node_id) + except ValueError: + return "" + + def _module_key(self, node_id: str) -> str: + """The repo-relative ``.go`` path a node id sits under — the symbol-table key. + + A Go node id is ``can:////go//``, so the file key is everything + between the language segment and the first declaration segment. A stand-in's id holds + ``@external`` there instead, which is not a path, so this raises and callers substitute + ``""`` rather than carrying a fictional file. + """ + prefix = self._scope_prefix + "go/" + if not node_id.startswith(prefix): + raise ValueError(f"{node_id!r} is not under this Go application") + rest = node_id[len(prefix) :] + head = rest.split("/", 1)[0] + if head.startswith("@"): + raise ValueError(f"{node_id!r} is a stand-in id and names no module") + # The file key runs to the last path segment ending in a source suffix. + parts = rest.split("/") + for i, seg in enumerate(parts): + if seg.endswith(".go"): + return "/".join(parts[: i + 1]) + raise ValueError(f"{node_id!r} carries no module path") + + #: A **callable** stand-in, told apart from an import ghost by carrying a ``signature``. + #: + #: ``:GoExternal`` is two things on this graph. A *callable* stand-in is a library function the + #: analyzer synthesized a body for (``@external/database/sql/DB.Query``, 35 of them on the + #: dataflow fixture); an *import ghost* is a package an unresolved import named + #: (``@external/bytes``, 16 more), reached only by ``GO_IMPORTS`` and carrying neither a + #: ``signature`` nor a ``module``. Only the first is a callable, and ``get_external_symbols`` + #: reported 51 against the local backend's 35 until this predicate separated them — measured + #: against the live graph. Keying on ``signature`` rather than on the id's shape is deliberate: + #: both live under ``@external/``, so the id cannot tell them apart. + _IS_STANDIN = "e.signature IS NOT NULL" + + @cached_property + def _external_signatures(self) -> Dict[str, str]: + """``{reported signature: can:// id}`` for every callable stand-in, cached.""" + rows = self._run( + f"MATCH (e:GoExternal) WHERE e.id STARTS WITH $pre AND {self._IS_STANDIN} " + "RETURN e.id AS id, e.module AS module, e.signature AS signature, e.name AS name ORDER BY e.id", + pre=self._scope_prefix, + ) + return {external_signature(r["module"] or "", r["signature"], r["name"] or ""): r["id"] for r in rows} + + def resolve_callable(self, name: str, *, in_class: str | None = None, in_module: str | None = None) -> SliceNode: + """The one callable ``name`` names (see :meth:`GoAnalysisBackend.resolve_callable`). + + The shared policy runs over the same unfiltered domain the local backend hands it, which is + the strongest form of "both backends resolve over the same set"; the Go receiver spellings + come from the shared :func:`~cldk.analysis.go.ids.callable_match_names`. + """ + sig = resolve_callable_signature(name, self._callable_candidates, in_class=in_class, in_module=in_module, by_full_name="more of the import path") + if sig in self._external_signatures: + ref = self._external_signatures[sig] + return SliceNode(file="", line=0, callable=sig, kind="external", name=sig.rpartition(".")[2], source=None, ref=ref) + rows = self._run( + "MATCH (c:GoCallable {signature:$sig}) WHERE c.id STARTS WITH $pre RETURN c.id AS ref, c.span_json AS span", + sig=sig, + pre=self._scope_prefix, + ) + if not rows: + raise KeyError(f"{sig!r} resolved but no :GoCallable node carries it in application {self.application_name!r}") + if len(rows) > 1: + raise ValueError(f"{sig!r} is carried by more than one analysed callable; neither can be addressed unambiguously") + ref = rows[0]["ref"] + span = json.loads(rows[0]["span"]) if rows[0]["span"] else None + return SliceNode( + file=self._path_of(ref), + line=span["start"][0] if span else 0, + callable=sig, + kind="callable", + name=sig.rpartition(".")[2], + source=None, + ref=ref, + ) + + def resolve_value(self, name: str, *, within: str) -> SliceNode: + """The one value of ``within`` that ``name`` names (see + :meth:`GoAnalysisBackend.resolve_value`). + + Two round trips, not one: the callable is resolved first, so an ambiguous ``within`` raises + naming *callables* rather than failing obscurely on a value search over a set of them — + through :func:`~cldk.analysis.commons.resolve.resolve_within`, so the advice it gives names + a keyword this method accepts. + + The ports are matched by the owner's **id**, not its signature: the owner may be a stand-in, + which carries no signature, and the id is what ``resolve_callable`` already returned. Each + port's ``b.var`` (the JSON's ``of``) is given every Go spelling by + :func:`~cldk.analysis.go.ids.go_value_names`, which is where the package-qualified form of a + global comes from (trap 3) — and reading ``of`` rather than indexing ``parameters`` is what + makes a method's receiver addressable (trap 2). + """ + owner = resolve_within(self.resolve_callable, within) + rows = self._run( + f"MATCH (c:{self._OWNER} {{id:$id}})-[:GO_HAS_BODY_NODE]->(b:GoBodyNode {{kind:'formal_in'}}) RETURN b.var AS var, b.id AS id ORDER BY b.id", + id=owner.ref, + ) + # A list, not a dict keyed by name: two values resolving to one name are a genuine ambiguity + # the policy must see, and a dict would silently keep the last row. + entries = [(r["id"], r["var"], nm) for r in rows if r["var"] for nm in go_value_names(r["var"])] + chosen = resolve_value_name(name, [nm for _i, _v, nm in entries], within=owner.callable) + node_id, var, _nm = next((i, v, nm) for i, v, nm in entries if nm == chosen) + kind, leaf, defined_in = body_node_kind("formal_in", var) + return SliceNode(file=owner.file, line=owner.line, callable=owner.callable, kind=kind, name=leaf, defined_in=defined_in, source=None, ref=node_id) + + # -----[ per-callable graphs ]----- + #: One owner's own ``rel`` edges. The second containment hop is a **pattern predicate**, not a + #: second pattern: written as a second match it would bind the same ``GO_HAS_BODY_NODE`` + #: relationship twice whenever ``s`` and ``d`` are the same node, and Cypher's + #: relationship-uniqueness rule would then drop the row — which is exactly the shape of a + #: data-dependence self-loop (a statement that reads a variable it also redefines). Losing those + #: silently, and counting ``total`` from the same match, is how a short page reports itself + #: complete (the Python-side bug #349, not repeated here). + _OWN_EDGES = ( + "MATCH (c:{owner}) WHERE c.id STARTS WITH $prefix AND c.signature = $sig " + "MATCH (c)-[:GO_HAS_BODY_NODE]->(s:GoBodyNode)-[r:{rel}]->(d:GoBodyNode) " + "WHERE (c)-[:GO_HAS_BODY_NODE]->(d) " + ) + + def _own_edges(self, name: str, in_class: str | None, rel: str, projection: str, order: EdgeOrder, page_size: int, cursor: str | None): + """One page of an owner's own ``rel`` edges: ``(signature, rows, whole size, is there more)``. + + ``page_size`` is validated **first**, before resolution, so a malformed argument is a + ``ValueError`` before any round trip — the order the local backend applies, so the two + cannot answer the same bad call with different exceptions. + + **Keyset, not ``SKIP``**: ``order.exprs`` is the canonical order written as Cypher (the same + components ``order.key`` produces in Python, held side by side on + :class:`~cldk.analysis.commons.bounds.EdgeOrder` so they cannot drift), and a cursor becomes + a ``WHERE`` filter rather than an offset, so a deep page costs what a shallow one does. + + The page asks for ``page_size + 1`` rows and reports ``more`` from whether it got them, so + "there is more" is a fact about the data rather than an inference from ``len(rows) == + page_size`` — which is wrong exactly when the set ends on a page boundary. + """ + check_page_size(page_size) + sig = self.resolve_callable(name, in_class=in_class).callable + match = self._OWN_EDGES.format(owner=self._OWNER, rel=rel) + params: Dict[str, Any] = {"prefix": self._scope_prefix, "sig": sig} + total = self._run(match + "RETURN count(r) AS total", **params)[0]["total"] + where = f"WHERE {keyset_where(order.exprs)} " if cursor is not None else "" + rows = self._run( + f"{match}WITH s.id AS src, d.id AS dst{projection} {where}RETURN * ORDER BY {', '.join(order.exprs)} LIMIT $lim", + lim=page_size + 1, + **params, + **(cursor_params(cursor, sig, len(order.exprs)) if cursor is not None else {}), + ) + return sig, rows[:page_size], total, len(rows) > page_size + + @staticmethod + def _page(model, scope: str, edges: List, order: EdgeOrder, total: int, more: bool) -> EdgePage: + """Wrap a page, deriving ``next_cursor`` from the *same* sort key the local backend uses — + so a cursor minted here and one minted there name the same position.""" + return EdgePage[model](edges=edges, total=total, next_cursor=encode_cursor(scope, order.key(edges[-1])) if more and edges else None) + + def get_cfg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoCfgEdge]: + """One page of control flow within one callable (see :meth:`GoAnalysisBackend.get_cfg`). + + The relationship is ``GO_CFG_NEXT``, not ``GO_CFG``: the Go projector names the control-flow + edge for the direction it points (``internal/neo4j/schema.go``'s ``RelCFGNext``), while the + four dependence relationships keep the bare ``sdg_rels`` spelling. + """ + sig, rows, total, more = self._own_edges(callable, in_class, "GO_CFG_NEXT", ", r.kind AS kind", CFG_ORDER, page_size, cursor) + return self._page(GoCfgEdge, sig, [GoCfgEdge(src=r["src"], dst=r["dst"], kind=r["kind"] or "fallthrough") for r in rows], CFG_ORDER, total, more) + + def get_cdg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoCdgEdge]: + """One page of control dependence within one callable (see :meth:`GoAnalysisBackend.get_cdg`).""" + sig, rows, total, more = self._own_edges(callable, in_class, "GO_CDG", "", CDG_ORDER, page_size, cursor) + return self._page(GoCdgEdge, sig, [GoCdgEdge(src=r["src"], dst=r["dst"]) for r in rows], CDG_ORDER, total, more) + + def get_ddg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoDdgEdge]: + """One page of data dependence within one callable (see :meth:`GoAnalysisBackend.get_ddg`). + + ``prov`` is projected pruned, so it is absent rather than null on an edge carrying none; + ``or []`` restores the model's default instead of failing validation, and the ``coalesce`` + in :data:`~cldk.analysis.go.backend.DDG_ORDER` does the same for the ordering. + """ + sig, rows, total, more = self._own_edges(callable, in_class, "GO_DDG", ", r.var AS var, r.prov AS prov", DDG_ORDER, page_size, cursor) + edges = [GoDdgEdge(src=r["src"], dst=r["dst"], var=r["var"], prov=list(r["prov"] or [])) for r in rows] + return self._page(GoDdgEdge, sig, edges, DDG_ORDER, total, more) + + # -----[ slicing and reachability ]----- + #: Variable-length Cypher, for the measured reason the Python backend records: Neo4j plans + #: ``(a)<-[:R*0..]-(b) RETURN DISTINCT b`` as a pruning breadth-first expansion, while driving + #: the same search one frontier at a time from Python is an order of magnitude slower and an + #: ``EXISTS`` subquery does not terminate at all (it enumerates trails instead of pruning). + #: + #: ``total`` and the page come back from one statement: the ids are collected in id order, + #: ``size()`` gives the whole slice's size, and only the first ``$cap`` are joined back to their + #: owners for hydration — collecting id *strings* rather than nodes is what keeps a large cone + #: inside the transaction's limits. + #: + #: The owner hop is ``OPTIONAL``: at L4 a reached vertex may belong to a stand-in, and an inner + #: join on ``:GoCallable`` would drop exactly the external hops that make an interprocedural + #: flow visible. + _SLICE = ( + "MATCH (r:GoBodyNode {{id:$id}}){left}[:{rels}*0..{depth}]{right}(m:GoBodyNode) " + "WITH DISTINCT m.id AS nid ORDER BY nid " + "WITH collect(nid) AS ids " + "WITH size(ids) AS total, ids[0..$cap] AS page " + "UNWIND page AS nid " + "MATCH (b:GoBodyNode {{id:nid}}) " + "OPTIONAL MATCH (c:{owner})-[:GO_HAS_BODY_NODE]->(b) " + "RETURN total, b.id AS ref, b.kind AS kind, b.var AS var, b.span_json AS span, " + "c.signature AS callable, c.span_json AS c_span" + ) + + def _slice(self, src: str, within: str, depth: int | None, max_nodes: int, *, backward: bool) -> Slice: + """One direction of :meth:`slice_backward` / :meth:`slice_forward`. + + The two differ only in which way the arrows point, so they share a query and a builder — a + second copy would be a second place for the node vocabulary to drift. + + Not scoped by the application prefix: a body-node id is stamped with its application, and + the projector only links nodes from its own run, so the traversal cannot leave the + application it started in. The seed is app-scoped by :meth:`resolve_value`. + """ + check_depth(depth) + check_max_nodes(max_nodes) + self._require_connected_ports("slice_backward" if backward else "slice_forward") + root = self.resolve_value(src, within=within) + query = self._SLICE.format( + owner=self._OWNER, + rels=SDG_REL_PATTERN, + depth="" if depth is None else depth, + left="<-" if backward else "-", + right="-" if backward else "->", + ) + rows = self._run(query, id=root.ref, cap=max_nodes) + nodes = [_slice_node(self._span_lines(r), self._module_key) for r in rows] + return Slice(nodes=nodes, roots=[root], resolved=slice_resolved([root]), total=rows[0]["total"] if rows else 0) + + @staticmethod + def _span_lines(row: Dict[str, Any]) -> Dict[str, Any]: + """``span_json``/``c_span`` turned into the ``line``/``c_line`` the shared row shape wants. + + The Go projector carries a span as one JSON string (a Bolt property stays scalar) where the + Python one carries ``start_line`` directly, so the start line is decoded here rather than in + :func:`_slice_node` — which is then the *same* function for both backends' row shapes. + """ + out = dict(row) + out["line"] = json.loads(row["span"])["start"][0] if row.get("span") else None + out["c_line"] = json.loads(row["c_span"])["start"][0] if row.get("c_span") else 0 + return out + + def slice_backward(self, src: str, *, within: str, depth: int | None = DEFAULT_DEPTH, max_nodes: int = DEFAULT_MAX_NODES) -> Slice: + """What affects this value (see :meth:`GoAnalysisBackend.slice_backward`).""" + return self._slice(src, within, depth, max_nodes, backward=True) + + def slice_forward(self, src: str, *, within: str, depth: int | None = DEFAULT_DEPTH, max_nodes: int = DEFAULT_MAX_NODES) -> Slice: + """What this value affects (see :meth:`GoAnalysisBackend.slice_forward`).""" + return self._slice(src, within, depth, max_nodes, backward=False) + + #: **The interior is pinned to ``:GoCallable``; the endpoints are not.** Every *source* of a + #: hop is a declared callable and prefix-scoped, which is exactly ``get_call_graph``'s edge set + #: — so a route cannot be manufactured through a node this application does not own. But the + #: walk's *end* may be a library stand-in, and must be: ``reaches("…closure_1", "DB.Query")`` + #: is the coarse form of the question the httpsql fixture gates, and pinning the endpoint to + #: ``:GoCallable`` answered ``False`` for a call that plainly exists (measured on the live + #: graph, where 69 of 247 ``GO_CALLS`` edges land on a stand-in). + #: + #: Admitting the endpoint cannot widen the interior, and that is a fact about the data rather + #: than a hope: **no stand-in has an outgoing ``GO_CALLS`` edge** — 0 of 247 on the live graph, + #: and 0 of 247 in the JSON the local backend reads, so the two agree. A stand-in is only ever + #: a terminus, so it can appear as ``(m)`` and never between two hops. Quantified path patterns + #: need Neo4j 5.9+. + _REACHES = ( + "MATCH (a:GoCallable {{id:$a}}) " + "MATCH (a) ((x:GoCallable)-[:GO_CALLS]->(y) WHERE x.id STARTS WITH $prefix){{1,{depth}}} (m) " + "WITH DISTINCT m WHERE m.id = $b RETURN count(m) > 0 AS ok" + ) + + def reaches(self, src: str, dst: str, *, depth: int | None = None) -> bool: + """Is there a call path (see :meth:`GoAnalysisBackend.reaches`)? + + Keyed by callable **id**, as the Go call graph is — so this needs no signature round trip, + and a stand-in endpoint (which has no signature) is addressable like any other. + """ + check_depth(depth) + a = self.resolve_callable(src).ref + b = self.resolve_callable(dst).ref + rows = self._run(self._REACHES.format(depth="" if depth is None else depth), a=a, b=b, prefix=self._scope_prefix) + return bool(rows and rows[0]["ok"]) + + #: ``{0,}``, so a sink with no callers is its own cone rather than an empty answer a caller + #: could not tell from "this name is wrong" (D7). Properties are projected into maps *before* + #: the cap, so only ``$cap`` of them cross the wire. + _CONE = ( + "MATCH (s:GoCallable) WHERE s.id IN $ids " + "MATCH (s) (()<-[:GO_CALLS]-(x:GoCallable) WHERE x.id STARTS WITH $prefix){{0,{depth}}} (m:GoCallable) " + "WITH DISTINCT m ORDER BY m.id " + "WITH collect({{signature: m.signature, name: m.signature, ref: m.id, span: m.span_json}}) AS found " + "RETURN size(found) AS total, found[0..$cap] AS page" + ) + + def backward_cone(self, sinks: Sequence[str], *, depth: int | None = DEFAULT_DEPTH, max_nodes: int = DEFAULT_MAX_NODES) -> Slice: + """Everything that can reach these sinks (see :meth:`GoAnalysisBackend.backward_cone`).""" + check_depth(depth) + check_max_nodes(max_nodes) + roots = cone_sinks(self.resolve_callable, sinks) + row = self._run( + self._CONE.format(depth="" if depth is None else depth), + ids=[r.ref for r in roots], + cap=max_nodes, + prefix=self._scope_prefix, + )[0] + nodes = [ + SliceNode( + file=self._path_of(n["ref"]), + line=json.loads(n["span"])["start"][0] if n["span"] else 0, + callable=n["signature"], + kind="callable", + name=(n["signature"] or "").rpartition(".")[2], + source=None, + ref=n["ref"], + ) + for n in row["page"] + ] + return Slice(nodes=nodes, roots=roots, resolved=slice_resolved(roots), total=row["total"]) + + # -----[ paths and the flow predicates ]----- + #: ``allShortestPaths`` and not a plain variable-length match: a variable-length pattern + #: enumerates *trails*, which does not terminate on a real dependence graph, while + #: ``allShortestPaths`` is a bidirectional breadth-first search. ``$cap`` is ``max_paths + 1``, + #: so one extra row reports truncation without a second counting traversal. + _PATHS = sdg_path_query( + "GO", + node_label="GoBodyNode", + projection="ref: n.id, kind: n.kind, var: n.var, span: n.span_json, " + "callable: head([(c:GoCallable|GoExternal)-[:GO_HAS_BODY_NODE]->(n) | c.signature]), " + "c_span: head([(c:GoCallable|GoExternal)-[:GO_HAS_BODY_NODE]->(n) | c.span_json])", + ) + + #: ``taint()``'s statement: the same shortest-path search as :attr:`_PATHS`, but m sources + #: against n sinks in one traversal, with the sanitizer cut **inside** the pattern and the cap + #: applied per pair. Sharing ``node_label`` and the projection verbatim is what makes a taint + #: witness and a ``paths_between`` witness describe a node identically. + _TAINT = sdg_taint_query( + "GO", + node_label="GoBodyNode", + projection="ref: n.id, kind: n.kind, var: n.var, span: n.span_json, " + "callable: head([(c:GoCallable|GoExternal)-[:GO_HAS_BODY_NODE]->(n) | c.signature]), " + "c_span: head([(c:GoCallable|GoExternal)-[:GO_HAS_BODY_NODE]->(n) | c.span_json])", + ) + + #: The variable names on SDG edges *leaving* a node inside ``$callable_prefix`` — ``startNode``, + #: the same end of the hop :attr:`_TAINT`'s cut predicate reads, so a sanitizer this validates + #: is one that predicate can actually match. **Delimited**, not a bare ``STARTS WITH``: the cut + #: it validates for is delimited, so a variable reachable only through an undelimited prefix (a + #: sibling callable whose id merely starts with this one — in Go, a closure of a same-named + #: outer function) would be accepted here and then sever nothing there. + _EDGE_VARS = ( + "MATCH (n:GoBodyNode)-[r:{rels}]->() " + "WHERE (n.id = $callable_prefix OR n.id STARTS WITH $callable_prefix + '@' OR n.id STARTS WITH $callable_prefix + '/') " + "RETURN collect(DISTINCT r.var) AS vars" + ) + + def _paths(self, query: str, node_of, a: SliceNode, b: SliceNode, *, src: str, dst: str, depth: int | None, max_paths: int) -> FlowPaths: + """Run one of the path queries and build the result. The ordering, the cap and the + completeness flag live here once.""" + check_distinct_endpoints(a, b) + rows = self._run(query.format(rels=SDG_REL_PATTERN, depth="" if depth is None else depth), src=src, dst=dst, cap=max_paths + 1, prefix=self._scope_prefix) + paths = [ + flow_path([node_of(self._span_lines(n), self._module_key) for n in r["ns"]], [(e["via"], e["var"], e["prov"]) for e in r["rs"]], via=VIA) + for r in rows[:max_paths] + ] + return FlowPaths(paths=paths, complete=len(rows) <= max_paths) + + def paths_between(self, src: str, dst: str, *, src_within: str, dst_within: str, depth: int | None = None, max_paths: int = DEFAULT_MAX_PATHS) -> FlowPaths: + """How a value reaches another value (see :meth:`GoAnalysisBackend.paths_between`).""" + check_depth(depth) + check_max_paths(max_paths) + self._require_connected_ports("paths_between") + a = self.resolve_value(src, within=src_within) + b = self.resolve_value(dst, within=dst_within) + return self._paths(self._PATHS, _slice_node, a, b, src=a.ref, dst=b.ref, depth=depth, max_paths=max_paths) + + #: ``WITH DISTINCT m`` before the membership test, so this plans as a pruning breadth-first + #: search rather than a trail enumeration. Not prefix-scoped: body-node ids embed the + #: application, so both the seed and every ``$dsts`` id are this application's by construction. + _VALUE_REACHES = "MATCH (a:GoBodyNode {{id:$src}})-[:{rels}*1..{depth}]->(m:GoBodyNode) WITH DISTINCT m WHERE m.id IN $dsts RETURN count(m) > 0 AS ok" + + #: Every value that *enters* an owner — its parameters, its receiver, and the globals and + #: captures it reads. Keyed by the owner's id, because a stand-in carries no signature. + _CALLEE_PORTS = f"MATCH (c:{_OWNER} {{id:$id}})-[:GO_HAS_BODY_NODE]->(b:GoBodyNode {{kind:'formal_in'}}) RETURN collect(b.id) AS ids" + + def _value_reaches(self, src: str, dsts: List[str], depth: int | None) -> bool: + """Does the value at ``src`` reach any of ``dsts``? The one predicate both flow questions + run, which is what makes ``flows_to_argument`` ⟹ ``flows_to_call`` a fact about their + *targets* rather than an agreement between two pieces of Cypher.""" + if not dsts: + return False + rows = self._run(self._VALUE_REACHES.format(rels=SDG_REL_PATTERN, depth="" if depth is None else depth), src=src, dsts=dsts) + return bool(rows and rows[0]["ok"]) + + def flows_to_call(self, src: str, callee: str, *, within: str, depth: int | None = None) -> bool: + """Does this value reach any value entering a call to ``callee`` + (see :meth:`GoAnalysisBackend.flows_to_call`)?""" + check_depth(depth) + self._require_connected_ports("flows_to_call") + root = self.resolve_value(src, within=within) + target = self.resolve_callable(callee).ref + ids = [i for i in self._run(self._CALLEE_PORTS, id=target)[0]["ids"] if i != root.ref] + return self._value_reaches(root.ref, ids, depth) + + def flows_to_argument(self, src: str, callee: str, arg: str, *, within: str, depth: int | None = None) -> bool: + """Does this value reach ``callee``'s ``arg`` + (see :meth:`GoAnalysisBackend.flows_to_argument`)?""" + check_depth(depth) + self._require_connected_ports("flows_to_argument") + root = self.resolve_value(src, within=within) + target = self.resolve_value(arg, within=callee).ref + return target != root.ref and self._value_reaches(root.ref, [target], depth) + + def _taint_walk( + self, + srcs: Sequence[SliceNode], + dsts: Sequence[SliceNode], + *, + cuts: List[Dict[str, str]], + cut_callables: List[str], + depth: int | None, + max_paths: int, + ) -> Tuple[List[Tuple[str, str, FlowPath]], Mapping[Tuple[str, str], List[Diagnostic]]]: + """The sanitized shortest walks, server-side (see :meth:`GoAnalysisBackend._taint_walk`). + + One statement for the whole batch, one row per witness — ``a.id AS src`` / ``b.id AS dst`` + carry the pairing back, because the m×n batching is only useful if the grouping survives it. + Ordering, the per-pair ``$cap`` and both cuts are the statement's, so nothing is re-sorted or + re-filtered here: a Python-side filter is the false-refutation bug + :func:`~cldk.analysis.commons.graphs.sdg_taint_query` exists to avoid, and a Python-side + sort would silently disagree with :func:`~cldk.analysis.commons.graphs.path_order`. + + The ledger comes back empty here as it does locally: Go's honesty signal is the + application-scope ``degraded[]``, which :meth:`taint` reads and files per pair. There is no + per-frontier signal to add, and inventing one from an unresolved call would void every + refutation in every application that calls a library. + """ + rows = self._run( + self._TAINT.format(rels=SDG_REL_PATTERN, depth="" if depth is None else depth), + srcs=[n.ref for n in srcs], + dsts=[n.ref for n in dsts], + cuts=cuts, + cut_callables=cut_callables, + cap=max_paths + 1, + prefix=self._scope_prefix, + ) + return [ + ( + r["src"], + r["dst"], + flow_path([_slice_node(self._span_lines(n), self._module_key) for n in r["ns"]], [(e["via"], e["var"], e["prov"]) for e in r["rs"]], via=VIA), + ) + for r in rows + ], {} + + def _edge_vars_in(self, callable_id: str) -> FrozenSet[str]: + """The edge variables scoped to this callable (see :meth:`GoAnalysisBackend._edge_vars_in`). + + Four of the five relationship types carry no ``var``, so the collected list holds ``None``; + those are dropped, because a membership test against a set holding ``None`` would answer a + question no caller can ask — ``resolve_sanitizers`` refuses a blank variable before it asks. + """ + rows = self._run(self._EDGE_VARS.format(rels=SDG_REL_PATTERN), callable_prefix=callable_id) + return frozenset(v for v in rows[0]["vars"] if v) + + # -----[ the honesty signals + hydration ]----- + def get_external_symbols(self) -> Dict[str, GoExternalSymbol]: + """The library stand-ins this graph carries (see + :meth:`GoAnalysisBackend.get_external_symbols`). + + Filtered to the **callable** stand-ins by :data:`_IS_STANDIN`: the label is shared with the + import ghosts an unresolved import mints, which are packages rather than callables and which + ``analysis.json`` does not carry under ``external_symbols`` at all. + """ + rows = self._run( + f"MATCH (e:GoExternal) WHERE e.id STARTS WITH $pre AND {self._IS_STANDIN} RETURN properties(e) AS p ORDER BY e.id", + pre=self._scope_prefix, + ) + return {r["p"]["id"]: R.external(r["p"]) for r in rows} + + def get_degraded(self) -> List[GoDegraded]: + """This graph's degradation markers (see :meth:`GoAnalysisBackend.get_degraded`). + + Off ``:GoApplication.degraded_json`` — the envelope-scope ``degraded`` list, carried as one + JSON string because a Bolt property stays scalar. A clean lazy push *clears* the property + rather than leaving a stale one, so an absent value is a positive "nothing degraded". + """ + rows = self._run("MATCH (a:GoApplication {id: $app_id}) RETURN a.degraded_json AS d", app_id=self._app_id) + return R.degraded(rows[0]["d"] if rows else None) + + def describe(self, nodes: Sequence[object]) -> List[SliceNode]: + """Fill in ``source`` for these positions (see :meth:`GoAnalysisBackend.describe`). + + One query for the batch rather than one per node. A vertex with no span gets ``None``: it is + a dataflow position, not a region of the file, and there is nothing to read on either + backend. + """ + out: List[SliceNode] = [] + for n in nodes: + if not isinstance(n, SliceNode): + continue + try: + out.append(n.model_copy(update={"source": self.get_source(n.ref)})) + except CodeanalyzerExecutionException: + out.append(n.model_copy(update={"source": None})) + return out + # -----[ artifact / dependency / config layer — reserved-but-unemitted on this projection ]----- def get_artifacts(self) -> Dict[str, PyArtifact]: raise CodeanalyzerExecutionException(ARTIFACT_LAYER_UNAVAILABLE.format(app=self.application_name)) diff --git a/cldk/analysis/go/neo4j/reconstruct.py b/cldk/analysis/go/neo4j/reconstruct.py index 19e0c2fa..cfbaf48b 100644 --- a/cldk/analysis/go/neo4j/reconstruct.py +++ b/cldk/analysis/go/neo4j/reconstruct.py @@ -32,9 +32,9 @@ from __future__ import annotations import json -from typing import Any, Dict, Mapping, Optional +from typing import Any, List, Mapping, Optional -from cldk.models.go import GoBodyNode, GoCallable, GoField, GoMetrics, GoModule, GoSpan, GoType +from cldk.models.go import GoBodyNode, GoCallable, GoDegraded, GoExternalSymbol, GoField, GoMetrics, GoModule, GoSpan, GoType Props = Mapping[str, Any] @@ -94,9 +94,52 @@ def field(props: Props) -> GoField: def body_node(props: Props) -> GoBodyNode: + """A :class:`GoBodyNode` from a ``:GoBodyNode`` node. + + **Two props are renamed on the wire** (``internal/neo4j/schema.go``), which is the whole reason + this function is not a splat: the JSON's ``of`` is projected as the property **``var``** (for + parity with Python's ``:PyBodyNode.var``), and the JSON's ``parent`` as **``call_node``**. + Reading them under their JSON names would silently leave every L4 vertex with ``of=None`` — and + ``of`` is what resolves a formal, so a backend reading it wrong would resolve no value at all. + """ return GoBodyNode( + id=props.get("id"), kind=props.get("kind", "call"), callee=props.get("callee"), + of=props.get("var"), + parent=props.get("call_node"), is_goroutine=bool(props.get("is_goroutine", False)), + is_deferred=bool(props.get("is_deferred", False)), span=_span(props), ) + + +def external(props: Props) -> GoExternalSymbol: + """A :class:`GoExternalSymbol` from a ``:GoExternal`` node. + + The node carries ``name``/``module``/``signature``/``path`` only; ``parameters``, ``receiver`` + and the stand-in ``body`` are not node properties — the body is the ``GO_HAS_BODY_NODE`` + relationship, which the backend follows where it needs the vertices. So a reconstructed + stand-in has those at their empty default rather than an invented value. + """ + return GoExternalSymbol( + id=props["id"], + kind=props.get("kind", "external"), + name=props.get("name", ""), + module=props.get("module", ""), + signature=props.get("signature"), + ) + + +def degraded(raw: str | None) -> List[GoDegraded]: + """The degradation markers from ``:GoApplication.degraded_json``. + + One JSON string on the wire, like ``span_json``: a Bolt property stays scalar, so the list is + carried encoded. Absent (a clean run) means ``[]`` — and the projector *clears* the property on + a clean lazy push rather than leaving a stale one (``internal/neo4j/bolt.go``'s + ``clearedAppProps``), so an absent value here is a positive statement that nothing degraded, + not merely an unknown. + """ + if not raw: + return [] + return [GoDegraded(**d) for d in json.loads(raw)] diff --git a/tests/analysis/go/test_go_neo4j_live.py b/tests/analysis/go/test_go_neo4j_live.py index 71bceb4b..c5e821dd 100644 --- a/tests/analysis/go/test_go_neo4j_live.py +++ b/tests/analysis/go/test_go_neo4j_live.py @@ -7,19 +7,29 @@ CLDK_TEST_GO_NEO4J_USER (default: neo4j) CLDK_TEST_GO_NEO4J_PASSWORD CLDK_TEST_GO_NEO4J_APP (the --app-name the graph was loaded with, e.g. go/cobra) + CLDK_TEST_GO_NEO4J_NAMESPACE (the --namespace; defaults to the app name) CLDK_TEST_GO_ANALYSIS_JSON (path to the SAME application's analysis.json, for the diff) With the first four set but no analysis.json, only the standalone (non-parity) checks run. + +**The L3/L4 parity checks (python-sdk#436) need a level-4 graph and a level-4 payload**, which +``--emit neo4j`` always is and ``analysis.json`` is only at ``-a 4``; they skip themselves on a +shallower pair rather than failing, since a level-2 payload legitimately has no dataflow to compare. +To reproduce the pair this suite was developed against:: + + cango -i /testdata/dataflow -a 4 -o /tmp/go-l4 --app-name dataflow --namespace dataflow + NEO4J_URI=bolt://localhost:7687 NEO4J_USERNAME=neo4j NEO4J_PASSWORD=... \\ + cango -i /testdata/dataflow --emit neo4j --app-name dataflow --namespace dataflow """ -import json import os import shutil from pathlib import Path import pytest -from cldk.analysis.commons.backend_config import Neo4jConnectionConfig +from cldk.analysis.commons.backend_config import cache_subdir +from cldk.analysis.commons.identity import resolve_identity from cldk.analysis.go.codeanalyzer import GoCodeanalyzer from cldk.analysis.go.neo4j import GoNeo4jBackend @@ -27,6 +37,7 @@ USER = os.environ.get("CLDK_TEST_GO_NEO4J_USER", "neo4j") PASSWORD = os.environ.get("CLDK_TEST_GO_NEO4J_PASSWORD") APP = os.environ.get("CLDK_TEST_GO_NEO4J_APP") +NAMESPACE = os.environ.get("CLDK_TEST_GO_NEO4J_NAMESPACE") or APP ANALYSIS_JSON = os.environ.get("CLDK_TEST_GO_ANALYSIS_JSON") pytestmark = pytest.mark.skipif( @@ -37,19 +48,107 @@ @pytest.fixture(scope="module") def neo4j_backend(): - b = GoNeo4jBackend(neo4j_uri=URI, neo4j_username=USER, neo4j_password=PASSWORD, application_name=APP) + b = GoNeo4jBackend(neo4j_uri=URI, neo4j_username=USER, neo4j_password=PASSWORD, application_name=APP, namespace=NAMESPACE) yield b b.close() @pytest.fixture(scope="module") def local_backend(tmp_path_factory): + """The local backend over the *same* application, fed from the cached ``analysis.json``. + + ``application_name``/``namespace`` are passed explicitly and must match the graph's: identity is + validated against the payload's own ``identity_namespace`` (#434), so letting them default from + the ``project_dir`` basename makes the cache look like a foreign application and the backend + re-runs ``cango`` against a path that does not exist. ``analysis_level`` is + ``system_dependency_graph`` so a level-4 payload is used at its full depth — the level only + selects what is *read*, since the cache is pre-seeded and no run happens. + """ if not (ANALYSIS_JSON and Path(ANALYSIS_JSON).exists()): pytest.skip("set CLDK_TEST_GO_ANALYSIS_JSON to the same application's analysis.json for the parity diff") cache = tmp_path_factory.mktemp("go-parity") - (cache / "go").mkdir() - shutil.copy(ANALYSIS_JSON, cache / "go" / "analysis.json") - return GoCodeanalyzer(project_dir="/unused", cache_dir=str(cache), analysis_level="call_graph", eager_analysis=False, target_files=None) + identity = resolve_identity(APP, NAMESPACE) + # ``cache_subdir`` keys the cache by the *identity digest* (#434), not by the bare language — + # ``/go/``. Seeding ``/go/analysis.json`` instead wrote + # where nothing reads, so the backend found no cache and tried to re-run cango. + seeded = cache_subdir(cache, None, "go", identity=identity) + seeded.mkdir(parents=True, exist_ok=True) + shutil.copy(ANALYSIS_JSON, seeded / "analysis.json") + return GoCodeanalyzer( + project_dir=None, + cache_dir=str(cache), + analysis_level="system_dependency_graph", + eager_analysis=False, + target_files=None, + application_name=APP, + namespace=NAMESPACE, + ) + + +@pytest.fixture(scope="module") +def dataflow_pair(local_backend, neo4j_backend): + """Both backends, once it is known they describe a level-4 application. + + A level-2 payload has no ``cfg``/``ddg`` and no ports, so a dataflow comparison over it would + be comparing two empty answers and would pass without measuring anything. Skipping is the + honest outcome; the gate is the *data* (do the ports carry dependence), never a version. + """ + if not local_backend._ports_carry_dependence: + pytest.skip("the analysis.json is below -a 4 (no interprocedural ports); the L3/L4 parity checks need one") + return local_backend, neo4j_backend + + +def _flowing_pair(local): + """A ``(caller, caller's plain formal_in, stand-in signature, its plain formal_in)`` quadruple + that a flow actually connects, or ``None``s. + + Searched rather than taken from the first external call edge: that edge's caller may read only a + package-level global (so it has no plain parameter to seed from), and its callee may be a + one-argument library function whose formal is a receiver. The search prefers a pair the local + backend reports a *path* for, so the comparison exercises witnesses and not just two empty + answers — and falls back to any well-formed pair, so a refutation is still compared where no + application flow exists. + """ + externals = {e.dst for e in local.get_application_view().call_graph if "/@external/" in e.dst} + fallback = None + for edge in local.get_application_view().call_graph: + if edge.dst not in externals or edge.src not in local._dataflow_owners: + continue + caller = local._dataflow_owners[edge.src][0] + callee = local._dataflow_owners.get(edge.dst, (None,))[0] + if callee is None: + continue + plain = lambda c: [n.of for n in (c.body or {}).values() if n.kind == "formal_in" and n.of and not n.of.startswith("<")] # noqa: E731 + src_names, dst_names = plain(caller), plain(callee) + if not src_names or not dst_names: + continue + callee_sig = local._callable_node(edge.dst).callable + for s in src_names: + for d in dst_names: + quad = (caller.signature, s, callee_sig, d) + fallback = fallback or quad + try: + if local.paths_between(s, d, src_within=caller.signature, dst_within=callee_sig, depth=None).paths: + return quad + except Exception: + continue + return fallback or (None, None, None, None) + + +def _one_method_and_its_param(local): + """A ``(callable signature, formal_in name)`` pair this application actually carries. + + Picked from the data rather than hardcoded, so the parity checks run on whatever application the + environment points at instead of only on the fixture they were written against. A callable with + both a ``ddg`` and a named ``formal_in`` is what every accessor below needs. + """ + for c, _tid, _path in local._iter_callables(): + if not c.ddg: + continue + for node in (c.body or {}).values(): + if node.kind == "formal_in" and node.of and not node.of.startswith((":", ":")): + return c.signature, node.of + return None, None # -----[ standalone: the graph answers at all ]----- @@ -91,3 +190,176 @@ def test_callees_match_on_a_real_callable(local_backend, neo4j_backend): lo = sorted((e.src, e.dst, tuple(e.prov), e.weight) for e in local_backend.get_callees(cid)) ne = sorted((e.src, e.dst, tuple(e.prov), e.weight) for e in neo4j_backend.get_callees(cid)) assert lo == ne + + +# -----[ L3/L4 parity (python-sdk#436): the dataflow surface answers identically ]----- +# +# Every check below compares the two backends rather than asserting a literal, which is the point: +# a literal would pin one application's numbers, while the contract is that two *different* +# implementations — an in-memory index over analysis.json and Cypher over the projected graph — +# cannot disagree. Four of these failed when first run against a live graph, and each failure was a +# real bug in the Cypher (the scope guard refusing stand-in ids, ``:GoExternal`` conflating callable +# stand-ins with import ghosts, and ``_REACHES`` pinning its endpoint to ``:GoCallable``); none was +# visible from the local backend alone, which is why they live here and not in a mocked suite. + + +def test_degradation_markers_match(dataflow_pair): + local, graph = dataflow_pair + assert [d.model_dump() for d in local.get_degraded()] == [d.model_dump() for d in graph.get_degraded()] + + +def test_external_standins_match(dataflow_pair): + """The *callable* stand-ins, which on the graph share the ``:GoExternal`` label with the import + ghosts an unresolved import mints — a package, not a callable, and absent from + ``analysis.json``'s ``external_symbols``. Conflating the two over-reported 51 against 35.""" + local, graph = dataflow_pair + assert set(local.get_external_symbols()) == set(graph.get_external_symbols()) + + +def test_ports_carry_dependence_agrees(dataflow_pair): + local, graph = dataflow_pair + assert local._ports_carry_dependence == graph._ports_carry_dependence + + +def test_per_callable_graphs_match(dataflow_pair): + local, graph = dataflow_pair + sig, _param = _one_method_and_its_param(local) + if sig is None: + pytest.skip("no callable with both a ddg and a named formal_in") + for getter in ("get_cfg", "get_cdg", "get_ddg"): + lo, ne = getattr(local, getter)(sig), getattr(graph, getter)(sig) + assert lo.total == ne.total, getter + shape = lambda p: [(e.src, e.dst, getattr(e, "kind", None), getattr(e, "var", None), tuple(getattr(e, "prov", []) or [])) for e in p.edges] # noqa: E731 + assert shape(lo) == shape(ne), getter + + +def test_edge_page_cursor_is_interchangeable(dataflow_pair): + """A cursor minted by one backend names the same position on the other — the property that + makes the two interchangeable mid-pagination, not merely equivalent per call.""" + local, graph = dataflow_pair + sig, _param = _one_method_and_its_param(local) + if sig is None: + pytest.skip("no callable with a ddg") + first = local.get_ddg(sig, page_size=1) + if first.next_cursor is None: + pytest.skip("that callable's ddg fits in one page") + lo = local.get_ddg(sig, page_size=1, cursor=first.next_cursor) + ne = graph.get_ddg(sig, page_size=1, cursor=first.next_cursor) + assert [(e.src, e.dst, e.var) for e in lo.edges] == [(e.src, e.dst, e.var) for e in ne.edges] + + +def test_value_resolution_matches(dataflow_pair): + local, graph = dataflow_pair + sig, param = _one_method_and_its_param(local) + if sig is None: + pytest.skip("no callable with a named formal_in") + shape = lambda v: (v.kind, v.name, v.defined_in, v.ref) # noqa: E731 + assert shape(local.resolve_value(param, within=sig)) == shape(graph.resolve_value(param, within=sig)) + + +def test_slices_match_in_both_directions(dataflow_pair): + local, graph = dataflow_pair + sig, param = _one_method_and_its_param(local) + if sig is None: + pytest.skip("no callable with a named formal_in") + for direction in ("slice_forward", "slice_backward"): + for depth in (None, 2): + lo = getattr(local, direction)(param, within=sig, depth=depth) + ne = getattr(graph, direction)(param, within=sig, depth=depth) + assert lo.total == ne.total, (direction, depth) + assert [n.ref for n in lo.nodes] == [n.ref for n in ne.nodes], (direction, depth) + assert [(n.kind, n.name, n.callable) for n in lo.nodes] == [(n.kind, n.name, n.callable) for n in ne.nodes], (direction, depth) + + +def test_a_standin_is_reachable_as_a_call_target(dataflow_pair): + """``reaches`` must admit a library stand-in as the walk's *end*. + + This is the coarse form of the HTTP→SQL question, and the Cypher answered ``False`` for it while + the local backend answered ``True``: the quantified pattern pinned ``(m)`` to ``:GoCallable``, + which a stand-in is not. Admitting the endpoint cannot widen the interior — no stand-in has an + outgoing ``GO_CALLS`` edge, asserted below so the premise is measured and not assumed. + """ + local, graph = dataflow_pair + edges = local.get_application_view().call_graph + assert not [e for e in edges if "/@external/" in e.src], "a stand-in gained an outgoing call edge; _REACHES' premise no longer holds" + into_standin = next((e for e in edges if "/@external/" in e.dst), None) + if into_standin is None: + pytest.skip("this application calls nothing outside itself") + caller = local._dataflow_owners[into_standin.src][0].signature + callee = local._callable_node(into_standin.dst).callable + assert local.reaches(caller, callee) is True + assert graph.reaches(caller, callee) is True + + +def test_flow_predicates_match_into_a_standin(dataflow_pair): + local, graph = dataflow_pair + caller_sig, src, callee_sig, arg = _flowing_pair(local) + if caller_sig is None: + pytest.skip("no caller/stand-in pair with a plain formal_in on both ends") + assert local.flows_to_call(src, callee_sig, within=caller_sig) == graph.flows_to_call(src, callee_sig, within=caller_sig) + assert local.flows_to_argument(src, callee_sig, arg, within=caller_sig) == graph.flows_to_argument(src, callee_sig, arg, within=caller_sig) + + +def test_paths_and_taint_match(dataflow_pair): + """The witnesses *and* the verdict: a path's hop sequence, and ``taint``'s per-pair + ``exhausted`` / ``complete`` ledger, must be byte-identical across the two.""" + local, graph = dataflow_pair + caller_sig, src, callee_sig, dst = _flowing_pair(local) + if caller_sig is None: + pytest.skip("no caller/stand-in pair with a plain formal_in on both ends") + + def path_shape(p): + return (len(p.paths), p.complete, [[(h.via, h.var, tuple(h.prov or []), h.to.ref) for h in path.hops] for path in p.paths]) + + lo = local.paths_between(src, dst, src_within=caller_sig, dst_within=callee_sig, depth=None) + ne = graph.paths_between(src, dst, src_within=caller_sig, dst_within=callee_sig, depth=None) + assert path_shape(lo) == path_shape(ne) + # The pair this suite selects is one the local backend finds a route for (see _flowing_pair), so + # a witness really was compared rather than two empty lists agreeing with each other. + assert lo.paths, "selected a pair with no path; the comparison above proved nothing" + + def taint_shape(t): + return (len(t.paths), sorted(t.exhausted), t.complete, t.resolved, len(t.unresolved)) + + sources, sinks = [(src, caller_sig)], [(dst, callee_sig)] + assert taint_shape(local.taint(sources=sources, sinks=sinks)) == taint_shape(graph.taint(sources=sources, sinks=sinks)) + # An explicit depth empties `exhausted` by rule, on both. + assert taint_shape(local.taint(sources=sources, sinks=sinks, depth=3)) == taint_shape(graph.taint(sources=sources, sinks=sinks, depth=3)) + # A variable cut is applied inside the search, not to the rows it returns, on both -- and it + # must actually bite, or this asserts that two unsanitized searches agree. + cut = [(src, caller_sig)] + lo_cut, ne_cut = local.taint(sources=sources, sinks=sinks, sanitizers=cut), graph.taint(sources=sources, sinks=sinks, sanitizers=cut) + assert taint_shape(lo_cut) == taint_shape(ne_cut) + assert not lo_cut.paths, "the variable cut severed nothing, so the sanitizer parity check is vacuous" + assert sorted(lo_cut.exhausted) == [(src, dst)] + + +def test_edge_vars_domain_matches(dataflow_pair): + """The domain a variable sanitizer is validated against. It must be the same set on both, or a + sanitizer accepted by one backend is refused by the other.""" + local, graph = dataflow_pair + sig, _param = _one_method_and_its_param(local) + if sig is None: + pytest.skip("no callable with a ddg") + ref = local.resolve_callable(sig).ref + assert sorted(local._edge_vars_in(ref)) == sorted(graph._edge_vars_in(ref)) + + +def test_miss_paths_match(dataflow_pair): + """Both backends must *fail* the same way, not merely succeed the same way: an ambiguous name + and an absent one are part of the contract, and a backend that raised a different type there + would be a drop-in replacement only on the happy path.""" + local, graph = dataflow_pair + sig, _param = _one_method_and_its_param(local) + if sig is None: + pytest.skip("no resolvable callable") + with pytest.raises(Exception) as lo: + local.resolve_callable("definitely_not_a_callable_in_this_app") + with pytest.raises(Exception) as ne: + graph.resolve_callable("definitely_not_a_callable_in_this_app") + assert type(lo.value) is type(ne.value) + with pytest.raises(Exception) as lo: + local.resolve_value("definitely_not_a_value", within=sig) + with pytest.raises(Exception) as ne: + graph.resolve_value("definitely_not_a_value", within=sig) + assert type(lo.value) is type(ne.value) From 962257727b4c8af1f93628410f1142f171cb80e5 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 5 Nov 2026 11:55:32 -0500 Subject: [PATCH 05/15] feat(go): expose the dataflow surface on the facade, and test it CLDK.go(...) now reaches the fifteen accessors, so the dataflow surface is usable without touching the backend. Also corrects CLDK.go's docstring, which still claimed cango produces levels 1 and 2 only. Two committed fixtures (tests/resources/go/dataflow, with a README on how each was produced and why): httpsql_l4.json is the analyzer's own HTTP->SQL fixture at -a 4, 116 KB, whose flow needs interface dispatch, interprocedural binding and the library stand-in all to be right; four_type_kinds.json is one declaration of each GoType.kind at -a 1, 3 KB. 50 tests over them, pinning behaviour a caller depends on rather than edge counts that move with every analyzer release: the gated flow is found, each of the four Go id/name traps is handled, and a refutation is only made where one is warranted (an explicit depth certifies nothing, a degraded analysis certifies nothing, an unreachable pair is refuted). Diagnostic.code gains `analysis_degraded`. It is a closed Literal and a published contract, so the addition is documented there; the name is language-neutral because "the analyzer told us it fell back" is not Go's alone. Refs #436 --- cldk/analysis/commons/results.py | 12 +- cldk/analysis/go/go_analysis.py | 125 ++++- tests/analysis/go/test_go_dataflow.py | 575 +++++++++++++++++++++ tests/resources/go/dataflow/README.md | 44 ++ tests/resources/go/dataflow/httpsql.go.txt | 95 ++++ tests/resources/go/dataflow/kinds.go.txt | 22 + 6 files changed, 869 insertions(+), 4 deletions(-) create mode 100644 tests/analysis/go/test_go_dataflow.py create mode 100644 tests/resources/go/dataflow/README.md create mode 100644 tests/resources/go/dataflow/httpsql.go.txt create mode 100644 tests/resources/go/dataflow/kinds.go.txt diff --git a/cldk/analysis/commons/results.py b/cldk/analysis/commons/results.py index e90db54d..36b38940 100644 --- a/cldk/analysis/commons/results.py +++ b/cldk/analysis/commons/results.py @@ -64,8 +64,9 @@ class Diagnostic(BaseModel): resolver, not in the error path", so nothing in the SDK constructs a ``Diagnostic`` with suggestions or with ``code="did_you_mean"``; the codes actually emitted are ``file_not_in_graph``, ``module_scope``, ``module_source_unavailable``, - ``entrypoint_report_unavailable``, and — since ``taint()`` — ``unresolved_dispatch`` - and ``degenerate_pair``. The field and the code stay because both are part + ``entrypoint_report_unavailable``, ``unresolved_dispatch`` and ``degenerate_pair`` + (since ``taint()``), and ``analysis_degraded`` (since Go's L3/L4 train, + python-sdk#436). The field and the ``did_you_mean`` code stay because both are part of a published model contract (``docs/agent-api-reference.md``) that a caller may already destructure; removing either is a separate, breaking change. """ @@ -84,6 +85,13 @@ class Diagnostic(BaseModel): "graph_schema_mismatch", "entrypoint_report_unavailable", "degenerate_pair", + # Part of the analysis was approximated, so an *absence* of flow is not a proof over it + # (python-sdk#436). Emitted by Go's ``taint()`` when ``get_degraded()`` is non-empty, and + # filed under every requested pair, which is what voids their ``exhausted`` certification. + # A language-neutral code rather than a Go-specific one: the condition it names — "the + # analyzer told us it fell back" — is not Go's alone, and a sibling that gains a + # degradation channel should report it under this name rather than minting a second. + "analysis_degraded", ] message: str suggestions: list[str] = [] diff --git a/cldk/analysis/go/go_analysis.py b/cldk/analysis/go/go_analysis.py index 903368b7..87144f5a 100644 --- a/cldk/analysis/go/go_analysis.py +++ b/cldk/analysis/go/go_analysis.py @@ -32,16 +32,31 @@ from __future__ import annotations from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Sequence, Tuple import networkx as nx from cldk.analysis.commons.identity import resolve_identity from cldk.analysis.commons.backend_config import CodeAnalyzerConfig, GoBackend, Neo4jConnectionConfig +from cldk.analysis.commons.bounds import DEFAULT_DEPTH, DEFAULT_MAX_NODES, DEFAULT_MAX_PATHS, DEFAULT_PAGE_SIZE +from cldk.analysis.commons.results import EdgePage, FlowPaths, Slice, SliceNode, TaintResult from cldk.analysis.go.backend import GoAnalysisBackend from cldk.analysis.go.codeanalyzer import GoCodeanalyzer from cldk.analysis.go.neo4j import GoNeo4jBackend -from cldk.models.go import GoApplication, GoCallable, GoCallGraphEdge, GoField, GoModule, GoParam, GoType +from cldk.models.go import ( + GoApplication, + GoCallable, + GoCallGraphEdge, + GoCdgEdge, + GoCfgEdge, + GoDdgEdge, + GoDegraded, + GoExternalSymbol, + GoField, + GoModule, + GoParam, + GoType, +) class GoAnalysis: @@ -159,3 +174,109 @@ def get_source(self, node_id: str) -> str: """The source text of ``node_id`` (a type, callable or field) — its span sliced from the owning module's source.""" return self.backend.get_source(node_id) + + # -----[ Tier B: dataflow (L3/L4) — the accessors share PythonAnalysis's names and + # signatures, so a caller's dataflow code ports between languages (python-sdk#436) ]----- + def resolve_callable(self, name: str, *, in_class: str | None = None, in_module: str | None = None) -> SliceNode: + """The one callable ``name`` names, as an addressable position. + + Accepts the receiver forms a Go programmer writes — ``Query``, ``DB.Query``, + ``(*DB).Query``, ``sql.DB.Query``, ``database/sql.DB.Query`` — and the **library stand-ins** + a level-4 analysis synthesizes, which is how ``database/sql.DB.Query`` is addressable as a + sink at all. Raises listing candidates rather than guessing when a name is ambiguous. + """ + return self.backend.resolve_callable(name, in_class=in_class, in_module=in_module) + + def resolve_value(self, name: str, *, within: str) -> SliceNode: + """The one value of ``within`` that ``name`` names — a parameter, a method's **receiver**, a + closure **capture**, or a package-level **global** the callable reads. + + A global is addressable by the spelling a Go programmer writes (``libflow.Last``) as well as + by its full import-path form; a bare leaf name that several packages export raises as the + ambiguity it is. + """ + return self.backend.resolve_value(name, within=within) + + def get_cfg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoCfgEdge]: + """One page of control flow within one callable (needs ``analysis_level`` 3 or deeper).""" + return self.backend.get_cfg(callable, in_class=in_class, page_size=page_size, cursor=cursor) + + def get_cdg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoCdgEdge]: + """One page of control dependence within one callable.""" + return self.backend.get_cdg(callable, in_class=in_class, page_size=page_size, cursor=cursor) + + def get_ddg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoDdgEdge]: + """One page of data dependence within one callable, each edge carrying the variable it + flows and its provenance tier (``reaching-defs`` / ``ssa`` / ``points-to``).""" + return self.backend.get_ddg(callable, in_class=in_class, page_size=page_size, cursor=cursor) + + def slice_backward(self, src: str, *, within: str, depth: int | None = DEFAULT_DEPTH, max_nodes: int = DEFAULT_MAX_NODES) -> Slice: + """Everything the value ``src`` depends on (needs ``analysis_level`` 4).""" + return self.backend.slice_backward(src, within=within, depth=depth, max_nodes=max_nodes) + + def slice_forward(self, src: str, *, within: str, depth: int | None = DEFAULT_DEPTH, max_nodes: int = DEFAULT_MAX_NODES) -> Slice: + """Everything the value ``src`` can affect.""" + return self.backend.slice_forward(src, within=within, depth=depth, max_nodes=max_nodes) + + def backward_cone(self, sinks: Sequence[str], *, depth: int | None = DEFAULT_DEPTH, max_nodes: int = DEFAULT_MAX_NODES) -> Slice: + """Every callable that can reach these sinks, over the call graph — the coarse pass triage + runs before asking for a value-level path.""" + return self.backend.backward_cone(sinks, depth=depth, max_nodes=max_nodes) + + def reaches(self, src: str, dst: str, *, depth: int | None = None) -> bool: + """Is there a call path from the callable ``src`` to the callable ``dst``?""" + return self.backend.reaches(src, dst, depth=depth) + + def paths_between(self, src: str, dst: str, *, src_within: str, dst_within: str, depth: int | None = None, max_paths: int = DEFAULT_MAX_PATHS) -> FlowPaths: + """How the value ``src`` reaches the value ``dst`` — the witnesses, shortest first. + + **Proves a flow; does not refute one.** An empty result cannot be told from "the flow left + the resolved graph" — :meth:`taint` is the accessor that reports that distinction. + """ + return self.backend.paths_between(src, dst, src_within=src_within, dst_within=dst_within, depth=depth, max_paths=max_paths) + + def flows_to_call(self, src: str, callee: str, *, within: str, depth: int | None = None) -> bool: + """Does the value ``src`` reach **any** value entering a call to ``callee``?""" + return self.backend.flows_to_call(src, callee, within=within, depth=depth) + + def flows_to_argument(self, src: str, callee: str, arg: str, *, within: str, depth: int | None = None) -> bool: + """Does the value ``src`` reach ``callee``'s parameter ``arg`` specifically?""" + return self.backend.flows_to_argument(src, callee, arg, within=within, depth=depth) + + def taint( + self, + sources: Sequence[Tuple[str, str]], + sinks: Sequence[Tuple[str, str]], + sanitizers: Sequence[Tuple[str, str] | str] = (), + *, + depth: int | None = None, + max_paths: int = DEFAULT_MAX_PATHS, + ) -> TaintResult: + """Which of these sources reach which of these sinks, and what to make of the ones that do not. + + m sources against n sinks in one traversal, reporting per pair: the witnesses in ``paths``, + the pairs an absence claim can be built on in ``exhausted``, and everything else in + ``unresolved``. The source/sink/sanitizer vocabulary is the caller's — no Go framework + catalogue ships. A non-empty :meth:`get_degraded` voids every ``exhausted`` certification, + because absence of a path is not a proof over an approximated graph. + """ + return self.backend.taint(sources, sinks, sanitizers, depth=depth, max_paths=max_paths) + + def get_external_symbols(self) -> Dict[str, GoExternalSymbol]: + """The library stand-ins a level-4 analysis synthesized, keyed by ``can://…/@external/…`` + id — where a path's external hops are explained. Not a dependency list.""" + return self.backend.get_external_symbols() + + def get_degraded(self) -> List[GoDegraded]: + """This analysis's degradation markers — empty when nothing was approximated. + + **Read this before trusting an absence.** Non-empty means some L4 step fell back to a + coarser result, so a path that *is* returned is still a real may-flow, but "no path" is not + a proof over the packages and callables it names. + """ + return self.backend.get_degraded() + + def describe(self, nodes: Sequence[object]) -> List[SliceNode]: + """Fill in ``source`` for these positions — a second call, because a slice answers *where* + and source answers *what*, and source is the one field with no size ceiling.""" + return self.backend.describe(nodes) diff --git a/tests/analysis/go/test_go_dataflow.py b/tests/analysis/go/test_go_dataflow.py new file mode 100644 index 00000000..8bdd8389 --- /dev/null +++ b/tests/analysis/go/test_go_dataflow.py @@ -0,0 +1,575 @@ +"""The Go L3/L4 dataflow surface, local backend, cango subprocess mocked — no binary, no Neo4j. + +Two committed payloads (``tests/resources/go/dataflow/``, see its README for how each was produced +and why): ``httpsql_l4.json`` is the analyzer's own HTTP→SQL fixture at ``-a 4``, whose flow is +gated analyzer-side by ``TestGateL4_HTTPToSQLPath``; ``four_type_kinds.json`` is one declaration of +each ``GoType.kind`` at ``-a 1``. + +**What is asserted and what is not.** These tests pin *behaviour a caller depends on* — that the +gated flow is found, that the four Go id/name traps are handled, that a refutation is only made +where one is warranted. They deliberately do not pin edge counts: those move with every analyzer +release and a test that fails on an improvement teaches nothing. Where a count appears it is +because the shape matters (a path's hop sequence, a cap's effect), not the magnitude. + +Backend *parity* is measured in ``test_go_neo4j_live.py``, which needs a live graph; that is where +the Cypher is checked, because four of its bugs were invisible from this side. +""" + +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from cldk import CLDK +from cldk.analysis import AnalysisLevel +from cldk.analysis.commons.backend_config import CodeAnalyzerConfig +from cldk.analysis.go.ids import body_node_id, callable_match_names, external_signature, go_value_names, split_body_node_id +from cldk.models.go import GoAnalysis +from cldk.utils.exceptions.exceptions import AmbiguousName, CodeanalyzerUsageException, SelectorNotInGraph + +FIXTURES = Path(__file__).resolve().parents[2] / "resources" / "go" / "dataflow" +HTTPSQL_L4 = FIXTURES / "httpsql_l4.json" +FOUR_KINDS = FIXTURES / "four_type_kinds.json" + +#: The sink the whole fixture exists to reach, and the two spellings of its one tainted argument. +SINK = "DB.Query" +SINK_ARG = "query" + + +def _fake_run_writing_output(payload: str): + """``subprocess.run`` that writes ``payload`` where cango's ``-o`` would put it.""" + + def _run(cmd, *args, **kwargs): + if "-o" in cmd: + out = Path(cmd[cmd.index("-o") + 1]) + out.mkdir(parents=True, exist_ok=True) + (out / "analysis.json").write_text(payload, encoding="utf-8") + return MagicMock(stdout=payload, returncode=0) + + return _run + + +def _facade(tmp_path, monkeypatch, payload_path, *, app, level=AnalysisLevel.system_dependency_graph): + monkeypatch.setenv("CODEANALYZER_GO_BIN", "cango") + monkeypatch.setattr("subprocess.run", _fake_run_writing_output(payload_path.read_text(encoding="utf-8"))) + return CLDK.go( + project_path=str(tmp_path), + analysis_level=level, + eager=True, + backend=CodeAnalyzerConfig(application_name=app, namespace=app, cache_dir=str(tmp_path)), + ) + + +@pytest.fixture +def httpsql(tmp_path, monkeypatch): + """The level-4 HTTP→SQL fixture, through the facade.""" + return _facade(tmp_path, monkeypatch, HTTPSQL_L4, app="httpsql") + + +@pytest.fixture +def backend(httpsql): + return httpsql.backend + + +# ====================================================================================== +# The parse blocker (#436's first commit): GoType.kind is a Literal, so a kind the SDK +# does not list fails the WHOLE document rather than dropping a field. +# ====================================================================================== + + +def test_all_four_type_kinds_parse(): + """``type Alias = int`` made ``GoAnalysis(**doc)`` raise, which blocked ``-a 1`` output from any + project declaring an alias — nothing to do with dataflow. All four kinds must load.""" + analysis = GoAnalysis.model_validate_json(FOUR_KINDS.read_text(encoding="utf-8")) + kinds = {ty.kind for module in analysis.application.symbol_table.values() for ty in module.types.values()} + assert kinds == {"struct", "interface", "defined", "alias"} + + +def test_a_fifth_kind_is_refused_loudly_rather_than_dropped(): + """The ``Literal`` is deliberate, not an oversight: a kind the analyzer adds must be a visible + failure here and not a value silently coerced or ignored, because every accessor keyed on + ``kind`` would then be quietly wrong. ``extra="ignore"`` does not cover a ``Literal``.""" + doc = json.loads(FOUR_KINDS.read_text(encoding="utf-8")) + module = next(iter(doc["application"]["symbol_table"].values())) + next(iter(module["types"].values()))["kind"] = "type_parameter" + with pytest.raises(Exception, match="kind"): + GoAnalysis(**doc) + + +def test_the_level_four_fixture_round_trips_losslessly(): + """Every overlay the dataflow surface reads must survive validation — the regression that + matters is the *silent* one, where a level-4 document parses and the graph is simply gone.""" + analysis = GoAnalysis.model_validate_json(HTTPSQL_L4.read_text(encoding="utf-8")) + app = analysis.application + assert analysis.max_level == 4 and analysis.k_limit is not None + assert app.param_in and app.param_out and app.external_symbols + callables = [c for m in app.symbol_table.values() for c in m.functions.values()] + assert any(c.cfg and c.cdg and c.ddg for c in callables), "no callable kept its L3 graphs" + assert any(c.summary for c in callables), "no callable kept its L4 summary edges" + vertices = [n for c in callables for n in (c.body or {}).values() if n.kind == "formal_in"] + assert vertices and all(v.id and v.of for v in vertices), "L4 vertices lost their id or `of`" + + +def test_absent_overlays_stay_none_not_empty(): + """``None`` and ``[]`` mean different things and the distinction is load-bearing: ``None`` is + "the analyzer did not compute this", ``[]`` is "it did, and there are no edges". Collapsing + them is what makes an empty answer indistinguishable from a proved absence (D7).""" + analysis = GoAnalysis.model_validate_json(FOUR_KINDS.read_text(encoding="utf-8")) + callables = [c for m in analysis.application.symbol_table.values() for c in m.functions.values()] + assert callables and all(c.cfg is None and c.ddg is None and c.summary is None for c in callables) + + +# ====================================================================================== +# The four Go-specific id/name traps (spec §7). Each is a rule that holds for Python or +# TypeScript and does NOT hold for Go, so a shared helper would be confidently wrong. +# ====================================================================================== + + +def test_trap1_a_standin_vertex_id_is_not_split_on_its_first_at(backend): + """A stand-in's *callable* id contains ``/@external/`` before the vertex ``@``, so + ``partition("@")`` yields a callable half that names nothing — and hands it onward as if it + did. Every vertex must recover its true owner, by prefix-stripping instead.""" + owners = list(backend._dataflow_owners) + standins = [e for e in backend.get_external_symbols().values() if e.body] + assert standins, "fixture carries no stand-in with a body" + naive_failures = 0 + for sym in standins: + for key, node in sym.body.items(): + assert split_body_node_id(node.id, owners) == (sym.id, key if key.startswith("@") else f"@{key}") + naive_failures += node.id.partition("@")[0] not in backend._dataflow_owners + assert naive_failures > 0, "the naive split would have worked here, so this proves nothing" + + +def test_trap1_a_nested_closure_does_not_lose_vertices_to_its_parent(backend): + """A callable id can be a proper *prefix* of a nested closure's (``…Handler`` and + ``…Handler.closure_1``), so the longest match is the right one — the shorter would claim the + longer's vertices.""" + owners = list(backend._dataflow_owners) + nested = [cid for cid in owners if ".closure_" in cid] + assert nested, "fixture carries no closures" + for cid in nested: + c = backend._dataflow_owners[cid][0] + for key in c.body or {}: + assert split_body_node_id(body_node_id(cid, key), owners)[0] == cid + + +def test_trap2_a_methods_receiver_resolves_by_of_not_by_parameter_ordinal(backend): + """``formal_in:0`` on a method is the *receiver*, which never appears in ``parameters``. A + backend pairing ``formal_in:N`` with ``parameters[N]`` would name every formal one position off, + silently — so the receiver is addressable and the next formal is still the first parameter.""" + method = backend._dataflow_owners[backend.resolve_callable("sqlRepo.Find").ref][0] + assert [p.name for p in method.parameters] == ["id"], "fixture changed shape" + receiver = (method.body or {})["@formal_in:0"] + assert receiver.of == "s" and receiver.of not in [p.name for p in method.parameters] + assert backend.resolve_value("s", within="sqlRepo.Find").ref.endswith("@formal_in:0") + assert backend.resolve_value("id", within="sqlRepo.Find").ref.endswith("@formal_in:1") + + +def test_trap3_a_global_is_matchable_by_the_spelling_a_go_programmer_writes(): + """``value_candidate`` renders a global's module half as the import path, whose separators are + ``/`` — and ``segment_match`` narrows on ``.``. So ``libflow.Last`` matched *nothing* until Go + added the package-qualified spelling as a second name for the same value.""" + from cldk.analysis.commons.resolve import segment_match + + names = go_value_names(":example.com/dataflow/libflow::Last") + assert names == ["example.com/dataflow/libflow.Last", "libflow.Last"] + # The canonical name stays first, so an error message and a SliceNode read in one vocabulary. + assert not segment_match("libflow.Last", names[0]), "the trap this exists for has gone away" + assert segment_match("libflow.Last", names[1]) + # A parameter and a capture have exactly one name, so this is the identity there. + assert go_value_names("id") == ["id"] + assert go_value_names(":buf") == ["buf"] + + +def test_trap4_a_degraded_analysis_cannot_certify_an_absence(backend, monkeypatch): + """``degraded[]`` is the honesty signal. A returned path over an approximated graph is still a + real may-flow, so this must **not** refuse — it must void the *absence* claims, which is what + ``exhausted`` carries. The clean fixture refutes an unreachable pair; the same call over a + degraded analysis refutes nothing.""" + from cldk.models.go import GoDegraded + + unreachable = dict(sources=[("id", "sqlRepo.Find")], sinks=[("user", "sqlRepo.Count")]) + clean = backend.taint(**unreachable) + assert clean.exhausted == [("id", "user")] and not clean.paths + + monkeypatch.setattr(type(backend), "get_degraded", lambda self: [GoDegraded(code="range_func", packages=["x"])]) + degraded = backend.taint(**unreachable) + assert not degraded.paths, "the flow did not change" + assert degraded.exhausted == [], "an approximated graph certified an absence" + assert degraded.complete is False + assert any("degrad" in d.message for d in degraded.unresolved), "no diagnostic says why" + # And a flow that IS found is still reported -- refusing would discard the findings that matter. + found = backend.taint(sources=[("id", "sqlRepo.Find")], sinks=[(SINK_ARG, SINK)]) + assert found.paths and found.exhausted == [] + + +# ====================================================================================== +# Addressing: the spellings spec §7 requires, and the ambiguities that must raise. +# ====================================================================================== + + +@pytest.mark.parametrize( + "spelling", + ["sqlRepo.Find", "(*sqlRepo).Find", "httpsql.sqlRepo.Find", "example.com/httpsql.sqlRepo.Find"], +) +def test_the_go_receiver_spellings_all_resolve(backend, spelling): + assert backend.resolve_callable(spelling).callable.endswith("httpsql.sqlRepo.Find") + + +def test_a_bare_method_name_shared_with_its_interface_is_ambiguous(backend): + """``Find`` names both the ``Repo`` interface's declaration and ``sqlRepo``'s implementation, + and they are different callables with different dataflow — the interface method has formal + vertices but no body. Picking either would be a guess, so the receiver is how a caller says + which, and ``in_class=`` narrows it the same way.""" + with pytest.raises(AmbiguousName) as e: + backend.resolve_callable("Find") + assert {c.rpartition(".")[0].rpartition(".")[2] for c in e.value.candidates} == {"Repo", "sqlRepo"} + assert backend.resolve_callable("sqlRepo.Find").callable.endswith("sqlRepo.Find") + assert backend.resolve_callable("Repo.Find").callable.endswith("Repo.Find") + + +@pytest.mark.parametrize("spelling", ["DB.Query", "sql.DB.Query", "database/sql.DB.Query"]) +def test_a_library_standin_is_addressable_as_a_sink(backend, spelling): + """Spec §7's own worked example. A stand-in is not a project callable, but its ``formal_in`` + ports are real L4 vertices a flow passes through — so it has to be in the resolution domain or + ``flows_to_call(…, "DB.Query")`` cannot be asked at all.""" + node = backend.resolve_callable(spelling) + assert node.callable == "database/sql.DB.Query" + assert node.kind == "external" and node.file == "", "a stand-in was given a fabricated position" + assert "/@external/" in node.ref + + +def test_an_ambiguous_name_raises_listing_candidates_rather_than_guessing(backend): + """``Query`` names both ``sql.DB.Query`` and ``url.URL.Query`` in this fixture. Picking one + would be the confident-wrong-answer failure the addressing layer exists to prevent.""" + with pytest.raises(AmbiguousName) as e: + backend.resolve_callable("Query") + assert len(e.value.candidates) >= 2 + assert all("Query" in c for c in e.value.candidates) + + +def test_a_name_that_matches_nothing_raises(backend): + with pytest.raises(SelectorNotInGraph): + backend.resolve_callable("NoSuchCallable") + with pytest.raises(SelectorNotInGraph): + backend.resolve_value("no_such_value", within="sqlRepo.Find") + + +def test_in_module_narrows_by_the_go_package_clause(backend): + """A Go module's readable name is its ``package`` clause. The shared derivation strips a ``.py`` + suffix and would leave ``httpsql.go`` riding into the dotted name, matching nothing.""" + assert backend.resolve_callable("sqlRepo.Find", in_module="httpsql").callable.endswith("sqlRepo.Find") + with pytest.raises(SelectorNotInGraph, match="in_module"): + backend.resolve_callable("sqlRepo.Find", in_module="not_a_package") + + +def test_two_packages_exporting_one_leaf_name_are_split_by_the_qualified_spelling(): + """Two globals sharing a leaf name are a real ambiguity, and the package-qualified spelling is + what resolves it — the capability trap 3 buys. + + Asserted on :func:`~cldk.analysis.go.ids.go_value_names` directly rather than through a facade: + the ``httpsql`` fixture reads no package-level globals, and manufacturing one would be a fixture + written to suit the test. The condition *was* measured end to end on the analyzer's full + ``testdata/dataflow`` corpus, where ``libstore`` and ``wrap`` both export ``lastQuery`` and one + callable reads both — a bare ``lastQuery`` raises there and each qualified form resolves. + """ + a = go_value_names(":example.com/dataflow/libstore::lastQuery") + b = go_value_names(":example.com/dataflow/wrap::lastQuery") + # The leaf collides, so a bare name cannot pick -- and the qualified names do not collide, so + # naming the package always can. + assert a[0].rpartition(".")[2] == b[0].rpartition(".")[2] == "lastQuery" + assert a[-1] == "libstore.lastQuery" and b[-1] == "wrap.lastQuery" + assert set(a).isdisjoint(b) + + +# ====================================================================================== +# The per-callable graphs, and the level gate. +# ====================================================================================== + + +@pytest.mark.parametrize("getter", ["get_cfg", "get_cdg", "get_ddg"]) +def test_the_three_graphs_answer_with_absolute_endpoint_ids(httpsql, getter): + """Endpoints are the body nodes' own ``can://`` ids — the spelling the graph merges on and the + one ``get_source`` accepts — not the callable-local keys the payload carries.""" + page = getattr(httpsql, getter)("sqlRepo.SanitizedFind") + assert page.edges and page.total == len(page.edges) + for e in page.edges: + assert e.src.startswith("can://") and e.dst.startswith("can://") + assert "SanitizedFind" in e.src + + +def test_the_ddg_carries_gos_three_provenance_tiers(httpsql): + """Unlike TypeScript's single tier, Go emits ``reaching-defs`` at L3 and adds ``ssa`` / + ``points-to`` at L4 — so ``prov_rank``'s certainty ordering is live here.""" + seen = {p for sig in ("sqlRepo.Find", "sqlRepo.SanitizedFind", "httpsql.Handler") for e in httpsql.get_ddg(sig).edges for p in e.prov} + assert "reaching-defs" in seen + assert seen - {"reaching-defs"}, "no L4 provenance tier present; is this payload really -a 4?" + + +def test_a_cfg_edge_keeps_its_kind_so_two_successors_stay_two_edges(httpsql): + page = httpsql.get_cfg("sqlRepo.SanitizedFind") + assert {e.kind for e in page.edges} - {"fallthrough"}, "no conditional edge kinds survived" + + +def test_paging_is_a_prefix_of_one_total_order(httpsql): + """A cursor is a position in the canonical order, so consecutive pages partition the edge set + without overlap or gap — and ``total`` is the whole size from the first page, not the page's.""" + whole = httpsql.get_ddg("sqlRepo.SanitizedFind") + assert whole.total > 2, "fixture too small to page" + first = httpsql.get_ddg("sqlRepo.SanitizedFind", page_size=2) + assert len(first.edges) == 2 and first.total == whole.total and first.next_cursor + second = httpsql.get_ddg("sqlRepo.SanitizedFind", page_size=2, cursor=first.next_cursor) + assert [(e.src, e.dst) for e in first.edges + second.edges] == [(e.src, e.dst) for e in whole.edges[:4]] + + +def test_a_shallow_analysis_refuses_rather_than_answering_empty(tmp_path, monkeypatch): + """At a level below the dataflow pass the three lists are absent, and an empty page there would + mean "not analysed" while looking exactly like "no dependence" (D7). So it raises, naming the + level to rebuild at.""" + shallow = _facade(tmp_path, monkeypatch, FOUR_KINDS, app="kinds", level=AnalysisLevel.symbol_table) + with pytest.raises(CodeanalyzerUsageException, match="level 3"): + shallow.get_cfg("kinds.Use") + + +def test_bad_bounds_are_refused_before_any_name_is_resolved(httpsql): + """The order of the checks is part of the contract: a caller with a malformed ``page_size`` + hears about ``page_size``, not about a name, and pays for no resolution to learn it.""" + with pytest.raises(ValueError): + httpsql.get_cfg("NoSuchCallableAtAll", page_size=0) + + +# ====================================================================================== +# The flow surface: the gated HTTP -> SQL question, and the refutations. +# ====================================================================================== + + +def test_the_gated_http_to_sql_flow_is_found(httpsql): + """``TestGateL4_HTTPToSQLPath``'s question, asked through the SDK. Interface dispatch, + interprocedural binding and the library stand-in all have to be right; any one missing yields + ``False``, and ``False`` is the answer that closes a live alert.""" + assert httpsql.flows_to_call("id", SINK, within="sqlRepo.Find") is True + assert httpsql.flows_to_argument("id", SINK, SINK_ARG, within="sqlRepo.Find") is True + + +def test_flows_to_argument_implies_flows_to_call(httpsql): + """A fact about their *targets* — one is the other narrowed to a single port — so the two can + never disagree.""" + for within in ("sqlRepo.Find", "sqlRepo.SanitizedFind", "sqlRepo.Count"): + arg_name = "id" if "Find" in within else "user" + if httpsql.flows_to_argument(arg_name, SINK, SINK_ARG, within=within): + assert httpsql.flows_to_call(arg_name, SINK, within=within) + + +def test_a_path_reads_as_a_sequence_of_justified_hops(httpsql): + """A path is a sequence, and each hop says what justified it in the *caller's* vocabulary — + ``data`` / ``argument`` / ``return`` / ``summary`` / ``control``, never ``GO_DDG``.""" + paths = httpsql.paths_between("id", SINK_ARG, src_within="sqlRepo.Find", dst_within=SINK, depth=None) + assert paths.paths and paths.complete + hops = paths.paths[0].hops + assert {h.via for h in hops} <= {"data", "control", "argument", "return", "summary"} + # It crosses into the callee, which is the hop an intraprocedural answer could not produce. + assert hops[-1].via == "argument" and "/@external/" in hops[-1].to.ref + assert hops[-1].to.kind == "parameter" and hops[-1].to.name == SINK_ARG + + +def test_max_paths_caps_witnesses_and_says_so(httpsql): + capped = httpsql.paths_between("id", SINK_ARG, src_within="sqlRepo.Find", dst_within=SINK, max_paths=1) + assert len(capped.paths) <= 1 + + +def test_the_two_slices_are_complementary(httpsql): + """Forward is what a value affects, backward what affects it. The sink's port must be in the + source's forward slice, and symmetrically.""" + fwd = httpsql.slice_forward("id", within="sqlRepo.Find", depth=None) + sink_ref = httpsql.resolve_value(SINK_ARG, within=SINK).ref + assert sink_ref in [n.ref for n in fwd.nodes] + back = httpsql.slice_backward(SINK_ARG, within=SINK, depth=None) + src_ref = httpsql.resolve_value("id", within="sqlRepo.Find").ref + assert src_ref in [n.ref for n in back.nodes] + + +def test_a_capped_slice_reports_the_whole_size(httpsql): + """A cap that fires is reported rather than silently returning less — ``total`` is the size of + the whole slice, which is what lets a caller see what it is walking into.""" + whole = httpsql.slice_forward("id", within="sqlRepo.Find", depth=None) + assert whole.total > 1, "fixture too small" + capped = httpsql.slice_forward("id", within="sqlRepo.Find", depth=None, max_nodes=1) + assert len(capped.nodes) == 1 and capped.total == whole.total and capped.complete is False + + +def test_a_slice_describes_a_standin_vertex_without_inventing_a_file(httpsql): + """A stand-in has no source file. ``kind="external"``/``file=""`` says so, rather than leaving + a fabricated path to be discovered as a sentinel later.""" + fwd = httpsql.slice_forward("id", within="sqlRepo.Find", depth=None) + external = [n for n in fwd.nodes if "/@external/" in n.ref] + assert external, "the flow never left the project, so this proves nothing" + assert all(n.file == "" for n in external) + + +def test_describe_hydrates_real_text_and_leaves_vertices_none(httpsql): + """Source is a second call because a slice answers *where* and source answers *what*. A + parameter-passing vertex is a dataflow position, not a region of the file, so it has no text — + ``None``, not ``""``. + + **A vertex is identified by its ``ref``, not by its ``kind``**, and that is the point worth + pinning: ``kind="return"`` is reported both for a ``return`` *statement* (which has a span, and + text) and for a ``formal_out`` *vertex* (which has neither), so a caller deciding from ``kind`` + alone would expect text where there is none. The id is what distinguishes them. + """ + nodes = httpsql.slice_forward("id", within="sqlRepo.Find", depth=2).nodes + hydrated = httpsql.describe(nodes) + assert any(n.source and "Query" in n.source for n in hydrated), "nothing hydrated" + is_vertex = lambda n: any(tag in n.ref for tag in ("formal_in", "formal_out", "actual_in", "actual_out", "@entry", "@exit")) # noqa: E731 + vertices = [n for n in hydrated if is_vertex(n)] + assert vertices, "the slice reached no parameter-passing vertex" + assert all(n.source is None for n in vertices), "a dataflow position was given source text" + statements = [n for n in hydrated if not is_vertex(n)] + assert statements and all(n.source for n in statements), "a real statement came back with no text" + # The same ``kind`` spans both groups, which is why the id and not the kind is the test. + assert {n.kind for n in vertices} & {n.kind for n in statements} + + +# ====================================================================================== +# taint: the verb triage needs. The rules are shared with the siblings (leg 4b, #382). +# ====================================================================================== + + +def test_taint_finds_the_flow_and_certifies_nothing_it_should_not(httpsql): + result = httpsql.taint(sources=[("id", "sqlRepo.Find")], sinks=[(SINK_ARG, SINK)]) + assert result.paths and result.complete + assert result.exhausted == [], "a pair with a witness was also certified absent" + assert "sqlRepo.Find" in result.resolved and "DB.Query" in result.resolved + + +def test_taint_refutes_a_pair_only_when_it_searched_to_exhaustion(httpsql): + """The distinction ``paths_between`` cannot report: an unreachable pair is *refuted*, and the + refutation is named by the strings the caller passed.""" + result = httpsql.taint(sources=[("user", "sqlRepo.Count")], sinks=[("id", "sqlRepo.Find")]) + assert not result.paths and result.exhausted == [("user", "id")] + + +def test_an_explicit_depth_empties_exhausted_by_rule(httpsql): + """A pair with no path inside N hops is not refuted, it is *unmeasured*. Conflating the two is + the bounded-boolean error, so a bounded search certifies nothing.""" + bounded = httpsql.taint(sources=[("user", "sqlRepo.Count")], sinks=[("id", "sqlRepo.Find")], depth=2) + assert bounded.exhausted == [] + unbounded = httpsql.taint(sources=[("user", "sqlRepo.Count")], sinks=[("id", "sqlRepo.Find")]) + assert unbounded.exhausted == [("user", "id")], "the pair is genuinely unreachable, so this is the control" + + +def test_max_paths_caps_per_pair_never_per_call(httpsql): + """With one sink and several sources a flat cap would let one prolific pair starve the rest, and + in triage the per-source witness *is* the answer.""" + sources = [("id", "sqlRepo.Find"), ("id", "sqlRepo.SanitizedFind"), ("user", "sqlRepo.Count")] + result = httpsql.taint(sources=sources, sinks=[(SINK_ARG, SINK)], max_paths=1) + witnesses = {(p.hops[0].frm.callable if p.hops else None) for p in result.paths} + assert len(result.paths) >= 2, "a flat cap would have returned one path for the whole batch" + assert len(witnesses) >= 2 + + +def test_a_variable_sanitizer_cuts_inside_the_search(httpsql): + """Both cuts are applied *inside* the traversal, not to the rows it returns: filtering afterwards + would report nothing for a source whose ten shortest paths are sanitized and whose eleventh is + not — a false refutation, the one output this accessor must never produce.""" + unsanitized = httpsql.taint(sources=[("id", "sqlRepo.Find")], sinks=[(SINK_ARG, SINK)]) + assert unsanitized.paths, "nothing to cut" + cut = httpsql.taint(sources=[("id", "sqlRepo.Find")], sinks=[(SINK_ARG, SINK)], sanitizers=[("id", "sqlRepo.Find")]) + assert not cut.paths and cut.exhausted == [("id", SINK_ARG)] + + +def test_a_sanitizer_naming_a_nonexistent_variable_is_refused(httpsql): + """The domain is the variables on SDG edges *scoped to that callable* — not the parameters, since + most real edge variables are locals. A sanitizer the caller believes is in force and is not would + be worse than an error.""" + with pytest.raises((SelectorNotInGraph, ValueError)): + httpsql.taint(sources=[("id", "sqlRepo.Find")], sinks=[(SINK_ARG, SINK)], sanitizers=[("no_such_var", "sqlRepo.Find")]) + + +def test_taint_refuses_an_empty_batch_rather_than_answering_nothing(httpsql): + with pytest.raises(ValueError, match="sources="): + httpsql.taint(sources=[], sinks=[(SINK_ARG, SINK)]) + with pytest.raises(ValueError, match="sinks="): + httpsql.taint(sources=[("id", "sqlRepo.Find")], sinks=[]) + + +def test_a_bare_string_batch_is_a_typeerror_not_a_silent_unpack(httpsql): + """``sources="id"`` would unpack into pairs of characters. Refusing it by type is the only way a + caller hears about the mistake rather than about a name that does not exist.""" + with pytest.raises(TypeError): + httpsql.taint(sources="id", sinks=[(SINK_ARG, SINK)]) + + +def test_two_selectors_naming_one_position_are_one_pair(httpsql): + """A pair is a pair of *resolved positions*, so a duplicated selector neither doubles the + witnesses nor the cap.""" + once = httpsql.taint(sources=[("id", "sqlRepo.Find")], sinks=[(SINK_ARG, SINK)]) + twice = httpsql.taint(sources=[("id", "sqlRepo.Find"), ("id", "sqlRepo.Find")], sinks=[(SINK_ARG, SINK)]) + assert len(twice.paths) == len(once.paths) + + +# ====================================================================================== +# The honesty signals, and the gates. +# ====================================================================================== + + +def test_the_standins_are_reported_with_readable_names(httpsql): + """Where a path's external hops are explained. The reported signature is assembled from the + stand-in's own ``module``/``signature``, not parsed back out of its ``can://`` id.""" + symbols = httpsql.get_external_symbols() + assert symbols + sigs = {external_signature(s.module, s.signature, s.name) for s in symbols.values()} + assert "database/sql.DB.Query" in sigs + query = next(s for s in symbols.values() if s.signature == "DB.Query") + assert query.body, "a stand-in with no body cannot be flowed through" + assert any(n.kind == "formal_in" for n in query.body.values()) + + +def test_a_clean_analysis_reports_no_degradation(httpsql): + assert httpsql.get_degraded() == [] + + +def test_the_port_gate_refuses_when_no_port_carries_dependence(tmp_path, monkeypatch): + """The interprocedural layer can be *present* and have nothing crossing into it, in which case + every slice is empty for want of edges rather than for want of a flow. The probe reads the data, + never a version — so a payload whose ports connect answers instead.""" + doc = json.loads(HTTPSQL_L4.read_text(encoding="utf-8")) + doc["application"]["param_in"] = [] + doc["application"]["param_out"] = [] + + def strip(owner): + """Empty every dependence list, closures included. + + Recursive because a Go closure is a callable in its own right with its own dataflow — and a + non-recursive strip left ``Handler.closure_1``'s edges in place, so a port still had an + outgoing edge and the probe correctly answered ``True``. The bug was in this setup, not in + the probe, which is the distinction a shallow strip hides. + """ + owner["ddg"], owner["cdg"], owner["summary"] = [], [], [] + for inner in (owner.get("callables") or {}).values(): + strip(inner) + + for module in doc["application"]["symbol_table"].values(): + for owner in list(module.get("functions", {}).values()) + [c for t in module.get("types", {}).values() for c in t.get("callables", {}).values()]: + strip(owner) + payload = tmp_path / "portless.json" + payload.write_text(json.dumps(doc), encoding="utf-8") + project = tmp_path / "p" + project.mkdir() + portless = _facade(project, monkeypatch, payload, app="httpsql") + assert portless.backend._ports_carry_dependence is False + with pytest.raises(Exception, match="dependence"): + portless.slice_forward("id", within="sqlRepo.Find") + with pytest.raises(Exception, match="dependence"): + portless.taint(sources=[("id", "sqlRepo.Find")], sinks=[(SINK_ARG, SINK)]) + + +def test_match_names_widen_resolution_without_multiplying_ambiguity(backend): + """``callable_match_names`` adds the package-qualified and pointer-receiver spellings. A + candidate matched by *any* of its names counts once, so these widen what resolves rather than + making more things ambiguous.""" + names = callable_match_names("example.com/httpsql.sqlRepo.Find") + assert names == ("example.com/httpsql.sqlRepo.Find", "httpsql.sqlRepo.Find", "(*sqlRepo).Find") + assert len(set(names)) == len(names), "a duplicate spelling would double-count a candidate" + # A signature with no slash has no package-qualified form to add. + assert callable_match_names("pkg.Fn") == ("pkg.Fn", "(*pkg).Fn") diff --git a/tests/resources/go/dataflow/README.md b/tests/resources/go/dataflow/README.md new file mode 100644 index 00000000..58010f7c --- /dev/null +++ b/tests/resources/go/dataflow/README.md @@ -0,0 +1,44 @@ +# Go L3/L4 dataflow fixtures (python-sdk#436) + +Two committed `cango` payloads, small enough to read, that between them exercise the whole +dataflow surface. The `.go.txt` files are the exact sources each was produced from — carried as +`.txt` so neither becomes part of this repo's own Go build, and so a reader can see what the +dataflow is *of* without cloning the analyzer. + +| File | Produced by | What it is for | +|------|-------------|----------------| +| `httpsql_l4.json` | `cango -i . -a 4 --app-name httpsql --namespace httpsql` | The flow surface. The analyzer's own `testdata/dataflow/httpsql` fixture, whose HTTP→SQL path is gated analyzer-side by `TestGateL4_HTTPToSQLPath`. | +| `four_type_kinds.json` | `cango -i . -a 1 --app-name kinds --namespace kinds` | The parse blocker. One declaration of each of the four `GoType.kind` values. | + +Regenerating either (after an analyzer release, say) is the command in the table, run from a +directory holding the matching `.go` file and a `go.mod`, with the output copied back here. +Produced against `cango 0.3.0`. + +## Why `httpsql` + +It is the smallest program that needs the *whole* L4 pipeline to answer one question. A handler +reads `r.URL.Query().Get("id")` and passes it to a `Repo` **interface** method; only VTA resolves +that call to `sqlRepo.Find`, which concatenates the value into `database/sql.DB.Query`'s `query` +argument. So a backend answering `flows_to_argument("id", "DB.Query", "query", …)` has to get +interface dispatch, interprocedural parameter binding *and* the library stand-in right — any one +of them missing yields `False`, and `False` is the answer that closes a live alert. + +It also carries, without being contrived to: + +- **a library stand-in as a sink** (`database/sql.DB.Query`), with a synthesized body a flow + passes *through* rather than stopping at; +- **an ambiguity** — `Query` names both `sql.DB.Query` and `url.URL.Query`, so a bare name must + raise rather than pick; +- **a sanitized sibling path** (`SanitizedHandler` → `SanitizedFind`, which parses the value with + `strconv.Atoi` first), so a refutation can be measured against a real alternative route and not + only against an absent one; +- **an unrelated pair** (`Count`/`user`, called only by `Audit`), so "no flow" can be checked + where no flow exists. + +## Why `four_type_kinds` + +`GoType.kind` is a `Literal`, which `extra="ignore"` does not cover: a kind the SDK does not list +is a hard parse failure of the whole document, not a dropped field. The analyzer emits four +(`internal/syntactic_analysis/symbol_table.go`), and the SDK listed three — so real `-a 1` output +from any project containing `type Alias = int` did not load at all. One declaration of each kind +is the cheapest guard against that recurring, and at `-a 1` the payload is ~3 KB. diff --git a/tests/resources/go/dataflow/httpsql.go.txt b/tests/resources/go/dataflow/httpsql.go.txt new file mode 100644 index 00000000..5f4055ee --- /dev/null +++ b/tests/resources/go/dataflow/httpsql.go.txt @@ -0,0 +1,95 @@ +// Package httpsql is the L4 "done" fixture (spec §8): an HTTP handler's +// request reaches database/sql's DB.Query through an interface call, and a +// second handler takes the same route through strconv.Atoi first. +package httpsql + +import ( + "database/sql" + "net/http" + "strconv" +) + +// Repo is the storage interface the handlers call; sqlRepo is its one +// implementation, so the interface calls bind sqlRepo's methods by VTA. +type Repo interface { + Find(id string) (*sql.Rows, error) + SanitizedFind(id string) (*sql.Rows, error) + Count(user string) (*sql.Rows, error) +} + +type sqlRepo struct{ db *sql.DB } + +// NewRepo returns the database-backed Repo. +func NewRepo(db *sql.DB) Repo { return &sqlRepo{db: db} } + +// Find builds the query from id: the sink. +func (s *sqlRepo) Find(id string) (*sql.Rows, error) { + return s.db.Query("SELECT name FROM users WHERE id = " + id) +} + +// SanitizedFind parses id as a number first and queries with the number. +func (s *sqlRepo) SanitizedFind(id string) (*sql.Rows, error) { + n, err := strconv.Atoi(id) + if err != nil { + return nil, err + } + return s.db.Query("SELECT name FROM users WHERE id = " + strconv.Itoa(n)) +} + +// Count counts a user's rows; only Audit calls it. +func (s *sqlRepo) Count(user string) (*sql.Rows, error) { + return s.db.Query("SELECT count(*) FROM events WHERE user = " + user) +} + +// Handler reads the id query parameter and looks it up. +func Handler(repo Repo) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id := r.URL.Query().Get("id") + rows, err := repo.Find(id) + if err != nil { + http.Error(w, "lookup failed", http.StatusInternalServerError) + return + } + rows.Close() + } +} + +// SanitizedHandler is Handler through SanitizedFind. +func SanitizedHandler(repo Repo) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id := r.URL.Query().Get("id") + rows, err := repo.SanitizedFind(id) + if err != nil { + http.Error(w, "lookup failed", http.StatusInternalServerError) + return + } + rows.Close() + } +} + +// Health answers without touching the database. +func Health(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + +// Audit is an entry point apart from Routes: it builds its own Repo inline +// (no call), so it shares no host variable and no in-project callee with the +// handlers; its only in-project callee is sqlRepo.Count. +func Audit(db *sql.DB, user string) error { + var repo Repo = &sqlRepo{db: db} + rows, err := repo.Count(user) + if err != nil { + return err + } + return rows.Close() +} + +// Routes wires the handlers to a database-backed Repo: without it no +// concrete type reaches the handlers' repo, and VTA binds repo.Find to +// nothing but the interface method. +func Routes(mux *http.ServeMux, db *sql.DB) { + repo := NewRepo(db) + mux.Handle("/user", Handler(repo)) + mux.Handle("/user/sanitized", SanitizedHandler(repo)) + mux.HandleFunc("/health", Health) +} diff --git a/tests/resources/go/dataflow/kinds.go.txt b/tests/resources/go/dataflow/kinds.go.txt new file mode 100644 index 00000000..b1fab5e1 --- /dev/null +++ b/tests/resources/go/dataflow/kinds.go.txt @@ -0,0 +1,22 @@ +// Package kinds declares one of each of the four type kinds the analyzer emits, +// which is what the SDK's GoType.kind Literal has to accept (python-sdk#436). +package kinds + +// S is a struct. +type S struct{ N int } + +// I is an interface. +type I interface{ M() int } + +// Celsius is a *defined* type (a new named type over an existing one). +type Celsius float64 + +// Alias is an *alias* (the same type under a second name) -- the kind that made +// GoAnalysis(**doc) raise before #436. +type Alias = int + +// M satisfies I. +func (s S) M() int { return s.N } + +// Use keeps every declaration reachable. +func Use(c Celsius, a Alias) int { return int(c) + a } From 5086d63c25017b529d73a43122bb811e91977718 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 5 Nov 2026 11:58:02 -0500 Subject: [PATCH 06/15] docs(go): record the L3/L4 surface in CLAUDE.md and the agent reference The language table's Go row said '-a 1..2 only' and 'no release cut yet'; both are stale. Adds the leg note for #436 with the four id/name traps and cango's three Neo4j property renames, and a Go section to the agent reference alongside TypeScript's and Java's. Every factual claim in the new section was checked against running code: the three DB.Query spellings, the receiver at formal_in:0 and absent from parameters[], the interface/implementation ambiguity, the level gate's message, the four depth defaults, and that all 16 documented accessors exist. Refs #436 --- CLAUDE.md | 26 ++++++++++++- docs/agent-api-reference.md | 78 +++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7bf0ef57..89a7e609 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ an optional read-only Neo4j backend — selected by the *type* of the `backend=` | Java | `CLDK.java(...)` (needs the `cldk[java]` extra) | `JCodeanalyzer` (the `codeanalyzer-java` 3.2.0 wheel's jar on its bundled JVM, subprocess, `-a 1..4` — no jar in this repo, no JDK download) | `JNeo4jBackend` (graphs emitted by ≥ 3.2.0, probed at attach) | `cldk/models/java/` (schema v2 mirror) + `projections.py` | | Python | `CLDK.python(...)` | `PyCodeanalyzer` (in-process `codeanalyzer-python`) | `PyNeo4jBackend` | re-exported from `codeanalyzer-python` | | TypeScript (+ JavaScript modules) | `CLDK.typescript(...)` | `TSCodeanalyzer` (`codeanalyzer-typescript` 1.6.0 binary from the wheel, subprocess; `-a 1..4`, but `--emit neo4j` takes no `-a` and is always full depth) | `TSNeo4jBackend` (graphs emitted by ≥ 1.5.2; older refused at attach) | `cldk/models/typescript/` (schema v2 mirror) | -| Go | `CLDK.go(...)` | `GoCodeanalyzer` (`cango` binary, subprocess, `-a 1..2` only — no L3/L4; `--analysis-schema 2` mandatory or it emits legacy v1; binary via `$CODEANALYZER_GO_BIN` or the `codeanalyzer-go` wheel — **no release cut yet, so the version pin is deferred**) | `GoNeo4jBackend` (lazy Cypher over the `GO_`-prefixed graph; **not** an eager rebuild — Go graphs reach enterprise scale) | `cldk/models/go/` (schema v2 mirror; **no `projections.py`** — out of SDK3 scope) | +| Go | `CLDK.go(...)` | `GoCodeanalyzer` (the `codeanalyzer-go` 0.3.0 wheel's `cango` binary, subprocess, `-a 1..4`; binary via `$CODEANALYZER_GO_BIN` or the wheel) | `GoNeo4jBackend` (lazy Cypher over the `GO_`-prefixed graph; **not** an eager rebuild — Go graphs reach enterprise scale) | `cldk/models/go/` (schema v2 mirror, L1–L4; **no `projections.py`** — out of SDK3 scope) | **Java, since leg 3a (#310):** the models are an `extra="forbid"` mirror of canonical schema v2, so a 1.x `analysis.json` (and a pre-3.0.1 Neo4j graph) is refused, not parsed; `get_call_graph()` keys @@ -45,6 +45,30 @@ absence claim in the result; `max_paths` caps per pair, never per call. The sour vocabulary is the caller's — no framework catalogue ships. Java shares the `formal_in` dependence probe above, so `taint` refuses on the same analyses the four verbs do. +**Go, since #436 (the L3/L4 train):** the dataflow surface, with `PythonAnalysisBackend`'s names and +signatures — `get_cfg`/`get_cdg`/`get_ddg`, `resolve_callable`/`resolve_value`, +`slice_backward`/`slice_forward`/`backward_cone`, `reaches`/`paths_between`, +`flows_to_call`/`flows_to_argument`, `taint`, `get_external_symbols`, `get_degraded`, `describe`. +`taint` is concrete on `GoAnalysisBackend`, so neither backend owns the policy; the port-connectivity +gate is Java's, declared on the ABC and implemented on both. Both backends answer identically +including the miss paths — measured on a live graph (`test_go_neo4j_live.py`), which is where four +Cypher bugs surfaced that the local side could not see. + +Four Go-specific traps live in **`cldk/analysis/go/ids.py`**, each a rule that holds for Python or +TypeScript and *not* for Go, so a shared helper would be confidently wrong rather than absent: +a body-node id must never be split on its first `@` (a stand-in's own id holds `/@external/` before +the vertex `@`; the naive split mis-owns every external vertex); a formal resolves by its own `of`, +never by ordinal into `parameters[]`, because a method's `formal_in:0` is the receiver; a global's +module half is slash-separated, so the spelling a Go programmer writes (`libflow.Last`) needs the +package-qualified name as a second match name; and `degraded[]` — **envelope** scope, beside +`application`, not on it — is the honesty signal, which voids every `exhausted` certification rather +than refusing (a returned path over an approximated graph is still a real may-flow). `cango`'s own +renames matter too: the Neo4j projection spells `of` as `var`, `parent` as `call_node`, and the CFG +relationship as `GO_CFG_NEXT`. The `GoType.kind` Literal carries all four kinds the analyzer emits +(`struct`/`interface`/`defined`/`alias`) — before #436 an `alias` failed the whole document, which +blocked even `-a 1`. `Diagnostic.code` gained `analysis_degraded` (language-neutral, closed Literal, +published contract). + The legacy `CLDK(language="").analysis(...)` entry still works as a compat shim. Adding a language means a new factory method + facade + backend ABC/impl(s) + models + tests — **update this table in the same change**. diff --git a/docs/agent-api-reference.md b/docs/agent-api-reference.md index 56394269..fbf22058 100644 --- a/docs/agent-api-reference.md +++ b/docs/agent-api-reference.md @@ -493,6 +493,84 @@ names its upstream issue where there is one: --- +## Go — what attaches today + +Everything outside this section is the **Python** surface. Go's type-centric navigation +(`get_types` / `get_type` / `get_functions` / `get_methods_of` / `get_method` / `get_fields`, +`get_callers` / `get_callees`, `get_source`) landed with the language; python-sdk#436 added the +dataflow surface below. + +```python +from cldk import CLDK +from cldk.analysis.commons.backend_config import CodeAnalyzerConfig, Neo4jConnectionConfig + +# In-process, over cango's analysis.json. Levels 1-4; 4 for anything on this page. +go = CLDK.go(project_path="/path/to/module", analysis_level="system_dependency_graph", + backend=CodeAnalyzerConfig()) + +# Against a deployed graph (read-only), keyed by the --app-name/--namespace it was emitted with +go = CLDK.go(backend=Neo4jConnectionConfig(uri="bolt://localhost:7687", username="neo4j", + password="…", application_name="my-app")) +``` + +`cango` comes from the `codeanalyzer-go` 0.3.0 wheel, or from `$CODEANALYZER_GO_BIN`. `--emit neo4j` +takes no `-a` and is always full depth, so **only a local analysis can be too shallow** — below +`-a 3` the per-callable graphs raise naming the level, and below `-a 4` a `resolve_value` finds no +port to address, because the port vertices do not exist there at all. + +**Landed for Go: the dataflow surface** — `get_cfg` / `get_cdg` / `get_ddg`, `resolve_callable` / +`resolve_value`, `slice_backward` / `slice_forward` / `backward_cone`, `reaches` / `paths_between`, +`flows_to_call` / `flows_to_argument`, `taint`, `get_external_symbols`, `get_degraded` and +`describe`. Python's signatures, keyword-for-keyword and default-for-default, with Go's edge models +(`GoCfgEdge` / `GoCdgEdge` / `GoDdgEdge`); the same bounds rule (slices bounded by default, +predicates and path queries unbounded, because a bounded predicate returns a *wrong* answer rather +than a small one) and the same `complete` protocol. Both backends answer identically including the +miss paths. + +Five Go-specific facts, each one a thing that will mislead you if you don't know it: + +- **A library stand-in is addressable, and is where most sinks live.** At `-a 4` the analyzer + synthesizes a body for a library callable, so `database/sql.DB.Query` is a resolvable callable + with real `formal_in` ports and a flow passes *through* it rather than stopping at the project + boundary. `resolve_callable("DB.Query")` works; so do `sql.DB.Query` and `database/sql.DB.Query`. + A stand-in resolves with `kind="external"` and `file=""` — it has no source file, and the SDK + reports that rather than inventing a path. `get_external_symbols()` is where their properties + live; it is **not** a dependency list. +- **A method's receiver is `formal_in:0`, and is not in `parameters[]`.** So the receiver is an + addressable value (`resolve_value("s", within="sqlRepo.Find")`) and the first declared parameter + is `formal_in:1`. Do not pair `formal_in:N` with `parameters[N]`: measured on the analyzer's + dataflow corpus, `formal_in:0.of` equals `parameters[0].name` on only 174 of 276 callables. An + interface method's receiver is spelled `_recv`; a concrete method's is the receiver variable's own + name. A closure's captures are `formal_in` vertices too, with no parameter at all. +- **A package-level global is addressed by the spelling you write.** `resolve_value("libflow.Last", + within=…)` resolves, as does the full import-path form `example.com/x/libflow.Last`; a bare + `lastQuery` that two packages both export raises `AmbiguousName` rather than picking, and the + package-qualified form is what splits them. +- **A bare method name shared with its interface is ambiguous, and that is correct.** `Find` names + both `Repo.Find` (the interface declaration — formal vertices, no body) and `sqlRepo.Find` (the + implementation). They have different dataflow, so the receiver is how you say which. +- **`get_degraded()` is the honesty signal — read it before trusting an absence.** Non-empty means + some L4 step fell back to a coarser result (`range_func`, `unmapped_ssa`, `no_vta`, …). A path + that *is* returned stays a real may-flow, so this does not refuse; what it does is void every + `exhausted` certification in a `taint` result and file an `analysis_degraded` diagnostic saying + why. `complete` goes `False` with it. + +**The port-connectivity gate.** `slice_*`, `paths_between`, `flows_to_*` and `taint` refuse — naming +the gap rather than returning a false empty — when no `formal_in` vertex in the attached analysis has +an outgoing dependence edge. Same rule as Java's, and the probe reads the *data*, never a version +string, so a payload or graph whose ports connect answers instead. + +**Known gap (codellm-devkit/codeanalyzer-go#30):** global → global `summary` edges are omitted. +Reachability is unchanged, but a depth-bounded walk may stop short where the shortcut is gone — one +more reason the predicates default to `depth=None`. + +**Scale caveat.** A local `-a 4` payload reaches 1.78 GB on Kubernetes and the local backend loads it +whole and builds the adjacency in memory. For an application that size, attach the Neo4j backend, +which pushes the same traversal into Cypher; `--emit neo4j` itself peaks at 21–25 GB on the largest +corpora (codellm-devkit/codeanalyzer-go#38). + +--- + ## The four moves Most questions decompose into these. Start here, then use the tables below. From 89b74119ed21fe3a28e384904ceff00289011607 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 5 Nov 2026 12:02:53 -0500 Subject: [PATCH 07/15] refactor(go): state the degradation notice once per batch A degraded analysis is a property of the whole analysis -- the approximation is somewhere in the SDG and nothing can say which pairs ran through it -- so it is one diagnostic under a key no pair claims, which is taint_verdict's own mechanism for a diagnostic it cannot attribute. The verdict is unchanged (nothing certified, batch not complete); filing per pair said the same thing m*n times, measured at 6 identical copies on a 3x2 batch and 400 on a 40x10. Refs #436 --- cldk/analysis/go/backend.py | 23 ++++++++++++++++------- tests/analysis/go/test_go_dataflow.py | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/cldk/analysis/go/backend.py b/cldk/analysis/go/backend.py index cfa3684b..79712db8 100644 --- a/cldk/analysis/go/backend.py +++ b/cldk/analysis/go/backend.py @@ -551,10 +551,18 @@ def taint( dsts = [self.resolve_value(name, within=within) for name, within in sinks] cuts, cut_callables = resolve_sanitizers(sanitizers, resolve_callable=self.resolve_callable, edge_vars_in=self._edge_vars_in) rows, blocked = self._taint_walk(srcs, dsts, cuts=cuts, cut_callables=cut_callables, depth=depth, max_paths=max_paths) - # A degraded analysis cannot certify an absence (trap 4). The diagnostic is filed under - # *every* requested pair, not under one key of this method's devising: ``taint_verdict`` - # reads the keys to decide ``exhausted``, and a key no requested pair claims would cost - # every pair its certification for want of anything able to attribute it. + # A degraded analysis cannot certify an absence (trap 4), and the condition is a property of + # the **whole analysis** rather than of any one pair: the approximation is somewhere in the + # SDG, and nothing here can say which pairs' routes ran through it. + # + # So it is filed under one key that **no requested pair claims**, which is exactly the + # mechanism :func:`~cldk.analysis.commons.graphs.taint_verdict` already has for a diagnostic + # it cannot attribute: an unclaimed key lands in ``unresolved`` and empties ``exhausted`` + # for the batch. That is the intended outcome here, not a side effect — a batch-wide + # approximation must cost every pair its certification. + # + # Filing it per pair instead gives the identical verdict and m*n copies of one message (400 + # of them on a 40x10 batch, measured), so the single key is the same answer said once. degraded = self.get_degraded() if degraded: codes = sorted({d.code for d in degraded}) @@ -563,11 +571,12 @@ def taint( message=( f"this analysis carries {len(degraded)} degradation marker(s) ({', '.join(codes)}), so part of the " "SDG is approximated: a path returned here is a real may-flow, but the absence of one is not a " - "proof. See get_degraded() for the affected packages and callables." + "proof, and no pair in this batch is certified. See get_degraded() for the affected packages " + "and callables." ), ) - per_pair = {(a.ref, b.ref): [note] for a in srcs for b in dsts} - blocked = {k: list(blocked.get(k, [])) + v for k, v in per_pair.items()} | {k: v for k, v in blocked.items() if k not in per_pair} + blocked = dict(blocked) + blocked[("", "")] = [note] return taint_verdict(sources, sinks, srcs, dsts, rows=rows, blocked=blocked, depth=depth, max_paths=max_paths) @abstractmethod diff --git a/tests/analysis/go/test_go_dataflow.py b/tests/analysis/go/test_go_dataflow.py index 8bdd8389..ea8c81bd 100644 --- a/tests/analysis/go/test_go_dataflow.py +++ b/tests/analysis/go/test_go_dataflow.py @@ -204,6 +204,28 @@ def test_trap4_a_degraded_analysis_cannot_certify_an_absence(backend, monkeypatc assert found.paths and found.exhausted == [] +def test_the_degradation_notice_is_stated_once_per_batch_not_once_per_pair(backend, monkeypatch): + """The condition is a property of the whole analysis — the approximation is somewhere in the + SDG and nothing can say which pairs ran through it — so it is one diagnostic, filed under a key + no pair claims, which is ``taint_verdict``'s own mechanism for a diagnostic it cannot attribute. + + The verdict is identical either way; filing per pair would say the same thing m×n times (400 + copies on a 40×10 batch), which buries the ledger a caller is meant to read. + """ + from cldk.models.go import GoDegraded + + monkeypatch.setattr(type(backend), "get_degraded", lambda self: [GoDegraded(code="no_vta")]) + batch = backend.taint( + sources=[("id", "sqlRepo.Find"), ("id", "sqlRepo.SanitizedFind"), ("user", "sqlRepo.Count")], + sinks=[(SINK_ARG, SINK), ("db", SINK)], + ) + assert len(batch.unresolved) == 1, "one batch-wide condition produced more than one diagnostic" + assert batch.unresolved[0].code == "analysis_degraded" + # The verdict is what matters, and it is the strict one: nothing certified, batch not complete. + assert batch.exhausted == [] and batch.complete is False + assert batch.paths, "the witnesses are still reported -- a degraded graph is not a refusal" + + # ====================================================================================== # Addressing: the spellings spec §7 requires, and the ambiguities that must raise. # ====================================================================================== From 2dee1a0f4f47620ba3c1c25140fbf115208986f6 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 5 Nov 2026 12:05:07 -0500 Subject: [PATCH 08/15] test(go): pin the admitted-id set of the widened scope guard #436 widened the Neo4j point-lookup guard to admit /@external/ beside /go/, since a library stand-in is this application's node. These pin what that did and did not open: the three own-node shapes are admitted, and all three foreign axes still refuse -- foreign namespace, foreign application (both separator-extended 'svc-extra' and bare 'svcX'), and sibling language -- as do the look-alike segments a prefix test could confuse ('gopher', '@externalish', and either bare segment with no trailing separator). Refs #436 --- .../test_go_neo4j_multi_application_scope.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/analysis/go/test_go_neo4j_multi_application_scope.py b/tests/analysis/go/test_go_neo4j_multi_application_scope.py index 7ad80739..04e386e1 100644 --- a/tests/analysis/go/test_go_neo4j_multi_application_scope.py +++ b/tests/analysis/go/test_go_neo4j_multi_application_scope.py @@ -82,3 +82,40 @@ def test_point_lookup_rejects_foreign_owner_or_language_before_query(uri): with pytest.raises(ValueError, match="scope"): backend.get_class(uri) assert len(driver.statements) == before + + +# python-sdk#436 widened the guard to admit `/@external/` alongside `/go/`, because a +# library stand-in is this application's node: the projector mints it under this prefix and it +# carries this application's L4 ports. The admitted set is the subject of these two tests -- the +# widening must not have opened an owner or language axis, and the cases below are the three axes +# plus the look-alike segments a prefix test could confuse. +@pytest.mark.parametrize( + "uri", + [ + "can://one/svc/go/server.go/T", # own project node + "can://one/svc/@external/database/sql/DB.Query", # own stand-in -- the widening + "can://one/svc/@external/database/sql/DB.Query@formal_in:1", # own stand-in's vertex + ], +) +def test_this_applications_own_nodes_are_admitted(uri): + backend = GoNeo4jBackend._from_driver(Driver(), application_name="svc", namespace="one") + assert backend._in_scope(uri) is True + + +@pytest.mark.parametrize( + "uri,why", + [ + ("can://two/svc/@external/bytes/Buffer.Read", "foreign namespace"), + ("can://one/svc-extra/@external/bytes/Buffer.Read", "foreign application, separator-extended"), + ("can://one/svcX/@external/bytes/Buffer.Read", "foreign application, no separator"), + ("can://one/svc/java/Foo.java/T", "sibling language"), + ("can://one/svc/typescript/a.ts/T", "sibling language"), + ("can://one/svc/gopher/server.go/T", "'go' as the prefix of a different segment"), + ("can://one/svc/@externalish/x", "'@external' as the prefix of a different segment"), + ("can://one/svc/@external", "the bare segment, no trailing separator"), + ("can://one/svc/go", "the bare language segment, no trailing separator"), + ], +) +def test_the_widened_guard_still_refuses_every_foreign_axis(uri, why): + backend = GoNeo4jBackend._from_driver(Driver(), application_name="svc", namespace="one") + assert backend._in_scope(uri) is False, why From 7f1bca36ad51d5115f7660250fb28da43fe49ade Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 5 Nov 2026 13:02:15 -0500 Subject: [PATCH 09/15] fix(go): make the two backends agree on a colliding signature and a stand-in Two bugs found by review against the live graph; both were backend disagreements invisible from either side alone. A Go signature is not always a unique address -- several files in one package may each declare their own init(), so one signature names two callables with different bodies (3 such collisions on the analyzer's own corpus). The local backend keyed its candidate list by signature, collapsing them into one entry and hiding the ambiguity from the shared policy: it raised a bare ValueError where the graph raised AmbiguousName, and in_module= naming a file narrowed on neither. Both now hand the policy one candidate per declaration, and ids.pick_declaration re-applies the caller's keywords to choose among the declarations behind the resolved signature -- which the Neo4j side previously discarded by re-looking-up on the bare signature and re-colliding. get_cfg/get_cdg/get_ddg on a library stand-in raised AttributeError locally (GoExternalSymbol has no cfg field) while the graph returned a clean empty page. Empty is the honest answer: a stand-in has no statements, only formal_in/formal_out ports. That is distinct from the level gate's D7 case, where empty would be indistinguishable from "no dependence" because the pass never ran; both the ABC docstring and the tests now say so. Verified on the live graph: 11 of 11 probes that differed now agree, and the 66-check parity suite still passes. Refs #436 --- cldk/analysis/go/backend.py | 23 ++++- cldk/analysis/go/codeanalyzer/codeanalyzer.py | 86 ++++++++++++------- cldk/analysis/go/ids.py | 49 ++++++++++- cldk/analysis/go/neo4j/neo4j_backend.py | 23 +++-- tests/analysis/go/test_go_dataflow.py | 62 +++++++++++++ tests/analysis/go/test_go_neo4j_live.py | 47 ++++++++++ 6 files changed, 247 insertions(+), 43 deletions(-) diff --git a/cldk/analysis/go/backend.py b/cldk/analysis/go/backend.py index 79712db8..a6423654 100644 --- a/cldk/analysis/go/backend.py +++ b/cldk/analysis/go/backend.py @@ -235,9 +235,19 @@ def resolve_callable(self, name: str, *, in_class: str | None = None, in_module: in_module: Disambiguate by module — the repo-relative ``.go`` path, or the package's dotted name. + **A Go signature is not always a unique address.** Several files in one package may each + declare their own ``init()``, so one signature can name several *different* callables with + different bodies and different dataflow (3 such collisions on the analyzer's own corpus). + Both backends hand the shared policy one candidate **per declaration**, so the ambiguity is + reported rather than silently resolved to whichever won, and ``in_module=`` naming one of + the declaring paths is what picks one. + Raises: AmbiguousName: More than one callable matched; the candidates are carried. SelectorNotInGraph: Nothing matched, or a keyword excluded every match. + ValueError: The resolved signature is declared in several files and the keywords given + do not single one out. The message names the paths, because passing one of them is + the way out. """ @abstractmethod @@ -292,11 +302,20 @@ def get_cfg(self, callable: str, *, in_class: str | None = None, page_size: int Endpoints are the body nodes' own ``can://`` ids, the spelling :meth:`get_source` accepts. + **An empty page for a library stand-in is an answer, not a gap.** ``callable`` may + resolve to one (``database/sql.DB.Query``), and a stand-in has no statements — the + analyzer synthesizes only its ``formal_in``/``formal_out`` ports — so all three graphs + are legitimately empty there. That is distinct from the level gate below, where empty + *would* be indistinguishable from "no dependence" because the pass never ran; here the + pass ran and the absence is a fact about what a stand-in is. Its ports are still real + vertices, so the slices and :meth:`taint` do cross it. + Raises: AmbiguousName: ``callable`` named more than one callable. SelectorNotInGraph: Nothing matched. - ValueError: ``page_size`` below 1, or ``cursor`` not from a previous page of this - accessor and this callable. + ValueError: ``page_size`` below 1, ``cursor`` not from a previous page of this accessor + and this callable, or the signature is declared in several files (see + :meth:`resolve_callable`). CodeanalyzerUsageException: (local backend) built below ``-a 3``. """ diff --git a/cldk/analysis/go/codeanalyzer/codeanalyzer.py b/cldk/analysis/go/codeanalyzer/codeanalyzer.py index daca5f94..77a8f10e 100644 --- a/cldk/analysis/go/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/go/codeanalyzer/codeanalyzer.py @@ -68,7 +68,7 @@ VIA, GoAnalysisBackend, ) -from cldk.analysis.go.ids import body_node_id, callable_match_names, external_signature, go_value_names +from cldk.analysis.go.ids import body_node_id, callable_match_names, external_signature, go_value_names, pick_declaration from cldk.models.go import ( GoAnalysis, GoApplication, @@ -526,7 +526,7 @@ def _slice_node(self, ref: str) -> SliceNode | None: # -----[ addressing ]----- @cached_property - def _candidates(self) -> Tuple[List[CallableCandidate], Dict[str, Tuple[object, str | None, str]], FrozenSet[str]]: + def _candidates(self) -> Tuple[List[CallableCandidate], Dict[str, List[Tuple[object, str | None, str]]]]: """``(candidates, {signature: owner}, colliding signatures)`` for :meth:`resolve_callable`. **The library stand-ins are in the domain**, and they have to be: spec §7's own worked @@ -542,28 +542,33 @@ def _candidates(self) -> Tuple[List[CallableCandidate], Dict[str, Tuple[object, than matching one through a path derivation that would be meaningless here (the same distinction Java's J-2 records). - Two callables sharing a signature collapse into one entry, and the resolver would then hand - back whichever won with no signal — so the collisions are recorded and a name resolving onto - one raises. Checked on the *answer* rather than eagerly, so a duplicate elsewhere does not - break every unrelated resolution. + **One candidate per declaration, not per signature.** A Go signature is not a unique + address: several files in one package may each declare their own ``init()``, so + ``example.com/x/initclosure.init`` names two different callables with different bodies (3 + such collisions on the analyzer's own corpus). Keying the candidate list by signature + collapsed those into one entry, which hid the ambiguity from the shared policy — so it + could neither report it (the Neo4j backend, whose rows are per declaration, raised + ``AmbiguousName`` where this raised a bare ``ValueError``) nor let ``in_module=`` narrow it. + Emitting one candidate per declaration is what makes the two backends hand the policy the + same domain, which is the only way they can agree on both the answer and the failure. + + ``declarations`` therefore maps a signature to *every* callable carrying it, and + :func:`~cldk.analysis.go.ids.pick_declaration` re-applies the caller's keywords to choose + among them once the policy has resolved the signature. """ - found: Dict[str, Tuple[object, str | None, str]] = {} - collisions: set = set() + declarations: Dict[str, List[Tuple[object, str | None, str]]] = {} for c, tid, path in self._iter_callables(): - if c.signature in found: - collisions.add(c.signature) - found[c.signature] = (c, tid, path) + declarations.setdefault(c.signature, []).append((c, tid, path)) cands = [ CallableCandidate(sig, tid, path, match_names=callable_match_names(sig), module_names=(pkg,) if (pkg := self._package_of(path)) else ()) - for sig, (_c, tid, path) in found.items() + for sig, entries in declarations.items() + for _c, tid, path in entries ] for sym in self.application.external_symbols.values(): sig = external_signature(sym.module, sym.signature, sym.name) - if sig in found: - collisions.add(sig) - found[sig] = (sym, None, "") + declarations.setdefault(sig, []).append((sym, None, "")) cands.append(CallableCandidate(sig, None, "", match_names=callable_match_names(sig), module_names=())) - return cands, found, frozenset(collisions) + return cands, declarations def _package_of(self, path: str) -> str | None: """The ``package`` clause of the module at ``path`` — what ``in_module=`` matches a Go @@ -585,11 +590,14 @@ def resolve_callable(self, name: str, *, in_class: str | None = None, in_module: "both backends resolve over the same set". The Go receiver spellings come from :func:`~cldk.analysis.go.ids.callable_match_names`, which is shared with the Neo4j backend. """ - cands, found, collisions = self._candidates + cands, declarations = self._candidates sig = resolve_callable_signature(name, cands, in_class=in_class, in_module=in_module, by_full_name="more of the import path") - if sig in collisions: - raise ValueError(f"{sig!r} is carried by more than one analysed callable; neither can be addressed unambiguously") - c, _tid, path = found[sig] + entries = declarations[sig] + # A signature can name several declarations (same-package multi-file ``init()``), so the + # keywords are re-applied to choose among them -- see ``pick_declaration`` for why the + # policy's own answer is one step short of an address here. + chosen = pick_declaration(sig, [(path, tid) for _c, tid, path in entries], in_class=in_class, in_module=in_module) + c, _tid, path = next((obj, tid, p) for obj, tid, p in entries if p == chosen) if not c.id: raise KeyError(f"{sig!r} carries no usable address (id={c.id!r})") # A stand-in has no file and no span, so it is described as an external rather than given a @@ -629,17 +637,31 @@ def resolve_value(self, name: str, *, within: str) -> SliceNode: return SliceNode(file=owner.file, line=owner.line, callable=owner.callable, kind=kind, name=leaf, defined_in=defined_in, source=None, ref=body_node_id(c.id, key)) # -----[ per-callable graphs ]----- - def _graphs_of(self, name: str, in_class: str | None, page_size: int) -> GoCallable: - """The callable ``name`` resolves to, once this backend is deep enough to have dataflow. + def _graphs_of(self, name: str, in_class: str | None, page_size: int) -> Tuple[str, GoCallable | None]: + """``(signature, callable)`` for ``name``, once this backend is deep enough to have dataflow. ``page_size`` is validated **first**, before the level guard and before resolution, so a malformed argument is a ``ValueError`` before anything else — the order the Neo4j backend applies too, so the two cannot answer the same bad call with different exceptions. + + **The callable is ``None`` for a library stand-in**, which is a real resolution target here + (spec §7's own example, ``database/sql.DB.Query``, is one) and has no intraprocedural flow + to report: the analyzer synthesizes only ``formal_in``/``formal_out`` ports for it, so + :class:`~cldk.models.go.GoExternalSymbol` carries no ``cfg``/``cdg``/``ddg`` field at all. + Returning ``None`` is what lets the three accessors answer with an **empty page**, which is + both the honest answer — the pass ran and a stand-in has no statements — and the answer the + Neo4j backend already gives, since its Cypher simply matches no rows on an external node. + Reading the field blindly raised ``AttributeError`` instead, which was neither. + + This is **not** the D7 case the level gate guards: there, empty would be indistinguishable + from "no dependence" because the pass never ran. Here the pass ran and the absence is a + fact about what a stand-in *is*. """ check_page_size(page_size) self._require_dataflow() node = self.resolve_callable(name, in_class=in_class) - return self._dataflow_owners[node.ref][0] + owner = self._dataflow_owners[node.ref][0] + return node.callable, owner if isinstance(owner, GoCallable) else None def get_cfg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoCfgEdge]: """One page of control flow within one callable (see :meth:`GoAnalysisBackend.get_cfg`). @@ -650,21 +672,21 @@ def get_cfg(self, callable: str, *, in_class: str | None = None, page_size: int the one the Neo4j backend would return. The join happens *before* the sort, because the order is over the ids a caller sees, not over the local keys. """ - c = self._graphs_of(callable, in_class, page_size) - edges = [GoCfgEdge(src=body_node_id(c.id, e.src), dst=body_node_id(c.id, e.dst), kind=e.kind) for e in c.cfg or []] - return edge_page(GoCfgEdge, c.signature, edges, CFG_ORDER, page_size, cursor) + sig, c = self._graphs_of(callable, in_class, page_size) + edges = [] if c is None else [GoCfgEdge(src=body_node_id(c.id, e.src), dst=body_node_id(c.id, e.dst), kind=e.kind) for e in c.cfg or []] + return edge_page(GoCfgEdge, sig, edges, CFG_ORDER, page_size, cursor) def get_cdg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoCdgEdge]: """One page of control dependence within one callable (see :meth:`GoAnalysisBackend.get_cdg`).""" - c = self._graphs_of(callable, in_class, page_size) - edges = [GoCdgEdge(src=body_node_id(c.id, e.src), dst=body_node_id(c.id, e.dst)) for e in c.cdg or []] - return edge_page(GoCdgEdge, c.signature, edges, CDG_ORDER, page_size, cursor) + sig, c = self._graphs_of(callable, in_class, page_size) + edges = [] if c is None else [GoCdgEdge(src=body_node_id(c.id, e.src), dst=body_node_id(c.id, e.dst)) for e in c.cdg or []] + return edge_page(GoCdgEdge, sig, edges, CDG_ORDER, page_size, cursor) def get_ddg(self, callable: str, *, in_class: str | None = None, page_size: int = DEFAULT_PAGE_SIZE, cursor: str | None = None) -> EdgePage[GoDdgEdge]: """One page of data dependence within one callable (see :meth:`GoAnalysisBackend.get_ddg`).""" - c = self._graphs_of(callable, in_class, page_size) - edges = [GoDdgEdge(src=body_node_id(c.id, e.src), dst=body_node_id(c.id, e.dst), var=e.var, prov=list(e.prov or [])) for e in c.ddg or []] - return edge_page(GoDdgEdge, c.signature, edges, DDG_ORDER, page_size, cursor) + sig, c = self._graphs_of(callable, in_class, page_size) + edges = [] if c is None else [GoDdgEdge(src=body_node_id(c.id, e.src), dst=body_node_id(c.id, e.dst), var=e.var, prov=list(e.prov or [])) for e in c.ddg or []] + return edge_page(GoDdgEdge, sig, edges, DDG_ORDER, page_size, cursor) # -----[ slicing and reachability ]----- def _reach(self, ref: str, direction: str, depth: int | None) -> set: diff --git a/cldk/analysis/go/ids.py b/cldk/analysis/go/ids.py index aef694da..abd2eb20 100644 --- a/cldk/analysis/go/ids.py +++ b/cldk/analysis/go/ids.py @@ -28,7 +28,7 @@ from typing import Iterable, List, Sequence, Tuple -from cldk.analysis.commons.resolve import value_candidate +from cldk.analysis.commons.resolve import segment_match, value_candidate #: The marker the analyzer puts in a library stand-in's **callable** id, before any vertex ``@``: #: ``can:////@external/bytes/Buffer.Read``. Its leading ``@`` is why @@ -196,6 +196,53 @@ def external_signature(module: str, signature: str, name: str) -> str: return f"{module}.{signature or name}" +def pick_declaration(sig: str, declarations: Sequence[Tuple[str, "str | None"]], *, in_class: "str | None", in_module: "str | None") -> str: + """The one declaration path of ``sig``, re-narrowed by the keywords that resolved it. + + **Why this exists.** Go lets several files in one package each declare their own ``init()``, so + one signature can name several *different* callables with different bodies and different + dataflow — ``example.com/x/initclosure.init`` is declared in both ``a.go`` and ``b.go`` on the + analyzer's own corpus (3 such collisions there). A signature is therefore not a unique address + in Go the way it is in Python, and both backends have to agree on that. + + :func:`~cldk.analysis.commons.resolve.resolve_callable_signature` returns a *signature string*, + which is one step short of an address when the signature collides: the keyword that narrowed the + candidate set is not re-applied to the declarations behind it, so a caller who wrote + ``in_module="a.go"`` and got past the policy would still face two declarations. This re-applies + exactly the policy's own predicates — :func:`segment_match` on the owning type for ``in_class``, + and on the repo-relative path with ``/`` for ``in_module`` — so a keyword that disambiguated the + candidates disambiguates the declaration too. + + Args: + sig: The signature the policy resolved to. + declarations: ``(path, owning type id)`` for every callable carrying ``sig``. + in_class / in_module: The keywords as the caller passed them. + + Returns: + The single matching declaration's path. + + Raises: + ValueError: ``sig`` is carried by more than one declaration and the keywords given do not + single one out. The message names the paths, because the way out is to pass one of + them — a caller cannot act on "it is ambiguous" alone. + """ + if len(declarations) == 1: + return declarations[0][0] + kept = list(declarations) + if in_class is not None: + kept = [(path, tid) for path, tid in kept if tid and segment_match(in_class, tid)] + if in_module is not None: + kept = [(path, tid) for path, tid in kept if segment_match(in_module, path, sep="/")] + if len(kept) == 1: + return kept[0][0] + paths = sorted(path for path, _tid in (kept or declarations)) + raise ValueError( + f"{sig!r} is declared in {len(paths)} files ({', '.join(paths)}) — Go allows several files in one " + f"package to declare the same signature, so it is not a unique address here. Narrow it with " + f"in_module= naming one of those paths." + ) + + def value_display(var: str) -> Tuple[str, str, "str | None"]: """``(kind, leaf, defined_in)`` for a ``formal_in`` vertex's ``var``, in the caller's vocabulary. diff --git a/cldk/analysis/go/neo4j/neo4j_backend.py b/cldk/analysis/go/neo4j/neo4j_backend.py index 0c3ce94d..afddfbc3 100644 --- a/cldk/analysis/go/neo4j/neo4j_backend.py +++ b/cldk/analysis/go/neo4j/neo4j_backend.py @@ -74,7 +74,7 @@ VIA, GoAnalysisBackend, ) -from cldk.analysis.go.ids import callable_match_names, external_signature, go_value_names +from cldk.analysis.go.ids import callable_match_names, external_signature, go_value_names, pick_declaration from cldk.analysis.go.neo4j import reconstruct as R from cldk.models.go import ( GoApplication, @@ -600,30 +600,37 @@ def resolve_callable(self, name: str, *, in_class: str | None = None, in_module: The shared policy runs over the same unfiltered domain the local backend hands it, which is the strongest form of "both backends resolve over the same set"; the Go receiver spellings come from the shared :func:`~cldk.analysis.go.ids.callable_match_names`. + + A signature that names several declarations (same-package multi-file ``init()``) is + narrowed by :func:`~cldk.analysis.go.ids.pick_declaration`, the same function and the same + keywords the local backend applies. Re-looking-up on the bare signature and refusing on + ``len(rows) > 1`` — what this did before — threw away the narrowing the policy had just + done, so ``in_module=`` could not disambiguate even when it had matched. """ sig = resolve_callable_signature(name, self._callable_candidates, in_class=in_class, in_module=in_module, by_full_name="more of the import path") if sig in self._external_signatures: ref = self._external_signatures[sig] return SliceNode(file="", line=0, callable=sig, kind="external", name=sig.rpartition(".")[2], source=None, ref=ref) rows = self._run( - "MATCH (c:GoCallable {signature:$sig}) WHERE c.id STARTS WITH $pre RETURN c.id AS ref, c.span_json AS span", + "MATCH (c:GoCallable {signature:$sig}) WHERE c.id STARTS WITH $pre " + "OPTIONAL MATCH (t:GoType)-[:GO_HAS_METHOD]->(c) " + "RETURN c.id AS ref, c.span_json AS span, t.id AS type_id ORDER BY c.id", sig=sig, pre=self._scope_prefix, ) if not rows: raise KeyError(f"{sig!r} resolved but no :GoCallable node carries it in application {self.application_name!r}") - if len(rows) > 1: - raise ValueError(f"{sig!r} is carried by more than one analysed callable; neither can be addressed unambiguously") - ref = rows[0]["ref"] - span = json.loads(rows[0]["span"]) if rows[0]["span"] else None + chosen = pick_declaration(sig, [(self._path_of(r["ref"]), r["type_id"]) for r in rows], in_class=in_class, in_module=in_module) + row = next(r for r in rows if self._path_of(r["ref"]) == chosen) + span = json.loads(row["span"]) if row["span"] else None return SliceNode( - file=self._path_of(ref), + file=chosen, line=span["start"][0] if span else 0, callable=sig, kind="callable", name=sig.rpartition(".")[2], source=None, - ref=ref, + ref=row["ref"], ) def resolve_value(self, name: str, *, within: str) -> SliceNode: diff --git a/tests/analysis/go/test_go_dataflow.py b/tests/analysis/go/test_go_dataflow.py index ea8c81bd..ffafb8a1 100644 --- a/tests/analysis/go/test_go_dataflow.py +++ b/tests/analysis/go/test_go_dataflow.py @@ -586,6 +586,68 @@ def strip(owner): portless.taint(sources=[("id", "sqlRepo.Find")], sinks=[(SINK_ARG, SINK)]) +# ====================================================================================== +# A Go signature is not always a unique address. Several files in one package may each +# declare their own init(), so one signature names several DIFFERENT callables with +# different bodies. Both bugs below were found by review against a live graph. +# ====================================================================================== + + +def test_a_signature_declared_in_several_files_is_an_ambiguity_not_a_pick(): + """Go permits a per-file ``init()``, so one signature can name two callables. + + The local backend keyed its candidate list by *signature*, which collapsed the two into one + entry and hid the ambiguity from the shared policy — so it raised a bare ``ValueError`` where + the Neo4j backend (whose rows are per declaration) raised ``AmbiguousName``, and ``in_module=`` + could not narrow it on either. One candidate per declaration is what makes both backends hand + the policy the same domain. + + Asserted on :func:`~cldk.analysis.go.ids.pick_declaration` directly: the committed fixtures are + single-file, and the condition was measured end to end on the analyzer's ``testdata/dataflow`` + corpus (3 colliding signatures there, `initclosure.init` / `initclosure.init.closure_1` / + `inits.init`), where both backends now answer identically. + """ + from cldk.analysis.go.ids import pick_declaration + + decls = [("initclosure/a.go", None), ("initclosure/b.go", None)] + # One declaration needs no narrowing. + assert pick_declaration("x.init", [("initclosure/a.go", None)], in_class=None, in_module=None) == "initclosure/a.go" + # Two do, and the error names the paths -- passing one is the way out, so "it is ambiguous" + # alone would be advice a caller cannot act on. + with pytest.raises(ValueError, match="declared in 2 files") as e: + pick_declaration("x.init", decls, in_class=None, in_module=None) + assert "initclosure/a.go" in str(e.value) and "initclosure/b.go" in str(e.value) + # The keyword that narrowed the candidates narrows the declaration too -- by bare file name or + # by the fuller path, since `in_module=` matches segment-wise on `/`. + assert pick_declaration("x.init", decls, in_class=None, in_module="a.go") == "initclosure/a.go" + assert pick_declaration("x.init", decls, in_class=None, in_module="initclosure/b.go") == "initclosure/b.go" + # A keyword that cannot split them leaves the ambiguity rather than picking. + with pytest.raises(ValueError, match="declared in 2 files"): + pick_declaration("x.init", decls, in_class=None, in_module="initclosure") + + +def test_the_three_graphs_answer_empty_for_a_library_standin(httpsql): + """A stand-in resolves (it is spec §7's own example) and has no statements — the analyzer + synthesizes only its ``formal_in``/``formal_out`` ports — so all three graphs are legitimately + empty there. + + The local backend read ``.cfg`` off the model blindly and raised ``AttributeError``, since + :class:`~cldk.models.go.GoExternalSymbol` has no such field, while the Neo4j backend returned a + clean empty page. An empty page is the honest answer, and it is **not** the D7 case the level + gate guards: there, empty would be indistinguishable from "no dependence" because the pass never + ran; here the pass ran and the absence is a fact about what a stand-in is. + """ + standin = httpsql.resolve_callable("DB.Query") + assert standin.kind == "external" + for getter in ("get_cfg", "get_cdg", "get_ddg"): + page = getattr(httpsql, getter)("DB.Query") + assert page.edges == [] and page.total == 0 and page.complete is True, getter + assert page.next_cursor is None, getter + # Its ports are still real vertices, so the flow surface DOES cross it -- which is the whole + # reason an empty page here must not read as "nothing to see". + assert httpsql.flows_to_argument("id", "DB.Query", "query", within="sqlRepo.Find") is True + + def test_match_names_widen_resolution_without_multiplying_ambiguity(backend): """``callable_match_names`` adds the package-qualified and pointer-receiver spellings. A candidate matched by *any* of its names counts once, so these widen what resolves rather than diff --git a/tests/analysis/go/test_go_neo4j_live.py b/tests/analysis/go/test_go_neo4j_live.py index c5e821dd..5a7f1c1c 100644 --- a/tests/analysis/go/test_go_neo4j_live.py +++ b/tests/analysis/go/test_go_neo4j_live.py @@ -345,6 +345,53 @@ def test_edge_vars_domain_matches(dataflow_pair): assert sorted(local._edge_vars_in(ref)) == sorted(graph._edge_vars_in(ref)) +def test_a_colliding_signature_fails_and_narrows_identically(dataflow_pair): + """Go permits a per-file ``init()``, so one signature can name two callables with different + bodies. Both backends must refuse it the same way *and* accept the same narrowing. + + This diverged: the local backend collapsed the declarations into one candidate and raised a bare + ``ValueError`` where the graph raised ``AmbiguousName``, and ``in_module=`` naming a file worked + on neither. Found by review against this graph; invisible from either backend alone. + """ + local, graph = dataflow_pair + by_sig = {} + for c, _tid, path in local._iter_callables(): + by_sig.setdefault(c.signature, []).append(path) + collisions = {sig: paths for sig, paths in by_sig.items() if len(paths) > 1} + if not collisions: + pytest.skip("this application declares no signature in more than one file") + sig, paths = sorted(collisions.items())[0] + + # Refused the same way, by type -- a bare name cannot pick between two real callables. + with pytest.raises(Exception) as lo: + local.resolve_callable(sig) + with pytest.raises(Exception) as ne: + graph.resolve_callable(sig) + assert type(lo.value) is type(ne.value), (type(lo.value), type(ne.value)) + + # And narrowed the same way, to the same node, by the file the caller names. + target = sorted(paths)[0] + lo_node = local.resolve_callable(sig, in_module=target) + ne_node = graph.resolve_callable(sig, in_module=target) + assert (lo_node.ref, lo_node.file, lo_node.callable) == (ne_node.ref, ne_node.file, ne_node.callable) + assert lo_node.file == target + + +def test_the_three_graphs_agree_on_a_standin(dataflow_pair): + """A stand-in has no statements, so all three graphs are empty — and both backends must *say* + so rather than one crashing. The local backend read a field ``GoExternalSymbol`` does not have. + """ + local, graph = dataflow_pair + standins = [s for s in local.get_external_symbols().values() if s.signature] + if not standins: + pytest.skip("this application calls nothing outside itself") + sig = f"{standins[0].module}.{standins[0].signature}" + for getter in ("get_cfg", "get_cdg", "get_ddg"): + lo, ne = getattr(local, getter)(sig), getattr(graph, getter)(sig) + assert (lo.total, list(lo.edges), lo.complete) == (ne.total, list(ne.edges), ne.complete), getter + assert lo.total == 0, f"{getter} on a stand-in returned edges; is it still body-less?" + + def test_miss_paths_match(dataflow_pair): """Both backends must *fail* the same way, not merely succeed the same way: an ambiguous name and an absent one are part of the contract, and a backend that raised a different type there From b04ec124a900fda4d9284e4bc9d7104a911a6d12 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Mon, 9 Nov 2026 11:40:45 -0500 Subject: [PATCH 10/15] refactor(go): pin the type-kind parse blocker beside the other model tests The alias fix stays bundled with this branch rather than shipping separately. Prompted by prototyping the standalone version, which surfaced two things worth keeping: The two kind tests assert a property of the model over -a 1 output, with nothing to do with the L3/L4 overlays, so they move from test_go_dataflow.py to test_go_analysis.py beside the other model tests. The second now pins ValidationError specifically rather than bare Exception, and says why the fix listed the missing kind instead of widening the field to str -- widening would have turned a loud failure into silence for every future kind. GoType's docstring now names the analyzer file rather than a line number that will drift, and states the consequence that makes this severe: an unlisted kind fails the whole document, not just the one type. Verified the regression test fails without the one-line fix and passes with it, so it measures the fix rather than merely accompanying it. Refs #436 --- cldk/models/go/models.py | 14 +++++---- tests/analysis/go/test_go_analysis.py | 41 +++++++++++++++++++++++++++ tests/analysis/go/test_go_dataflow.py | 24 ++-------------- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/cldk/models/go/models.py b/cldk/models/go/models.py index 673d5879..a521ab47 100644 --- a/cldk/models/go/models.py +++ b/cldk/models/go/models.py @@ -281,11 +281,15 @@ class GoType(_NullSafeBase): discriminated by ``kind`` rather than a ``Union`` of subclasses. ``base_types`` are embedded types; ``fields`` are the struct's data fields (empty for every other kind). - The four kinds the analyzer emits (``symbol_table.go:432``): ``struct``, ``interface``, - ``defined`` (``type Celsius float64``) and ``alias`` (``type Alias = int``). This is a - ``Literal`` rather than an open ``str`` so a *new* kind is a loud failure and not a silent - mis-read — ``extra="ignore"`` does not cover a ``Literal`` mismatch, which is exactly why the - narrower spelling refused to parse real ``-a 1`` output (python-sdk#436). + The four kinds the analyzer emits (``internal/syntactic_analysis/symbol_table.go``): + ``struct``, ``interface``, ``defined`` (``type Celsius float64``) and ``alias`` + (``type Alias = int``). + + This stays a ``Literal`` rather than an open ``str`` so a *new* kind is a loud failure and not + a silent mis-read — but note what that costs: ``extra="ignore"`` does not cover a ``Literal`` + mismatch, so an unlisted kind fails the **whole document**, not just the one type. That is why + omitting ``alias`` refused real ``-a 1`` output from any project declaring a type alias + (python-sdk#436), independently of the analysis level or the dataflow overlays. """ id: str diff --git a/tests/analysis/go/test_go_analysis.py b/tests/analysis/go/test_go_analysis.py index f41fbb5f..e1ed30c3 100644 --- a/tests/analysis/go/test_go_analysis.py +++ b/tests/analysis/go/test_go_analysis.py @@ -270,3 +270,44 @@ def test_real_corpus_call_graph_dangling_free(litestream_facade): assert g.number_of_edges() == 3082 for src, dst in g.edges(): assert "id" in g.nodes[src] and "id" in g.nodes[dst] + + +# -----[ GoType.kind accepts every kind the analyzer emits ]----- +# +# ``kind`` is a ``Literal``, and ``extra="ignore"`` does not cover a ``Literal`` mismatch: an +# unlisted kind fails the WHOLE document rather than dropping one field. So the listed set has to +# be exactly the analyzer's, and a gap there is not a cosmetic omission — it is a project that does +# not load at all. These live here rather than with the dataflow tests because the fact is about +# ``-a 1`` output and the model that mirrors it, not about the L3/L4 overlays. +FOUR_KINDS = Path(__file__).resolve().parents[2] / "resources" / "go" / "dataflow" / "four_type_kinds.json" + + +def test_all_four_type_kinds_parse(): + """``type Alias = int`` and ``type Celsius float64`` are ordinary Go, and cango emits them as + ``alias`` / ``defined`` (``internal/syntactic_analysis/symbol_table.go``). Listing only + ``struct``/``interface``/``defined`` made ``GoAnalysis`` refuse real ``-a 1`` output from any + project declaring an alias — nothing to do with the analysis level or the dataflow overlays. + """ + from cldk.models.go import GoAnalysis + + analysis = GoAnalysis.model_validate_json(FOUR_KINDS.read_text(encoding="utf-8")) + kinds = {ty.kind for module in analysis.application.symbol_table.values() for ty in module.types.values()} + assert kinds == {"struct", "interface", "defined", "alias"} + + +def test_an_unknown_fifth_kind_still_fails_loudly(): + """The ``Literal`` is a deliberate choice, not an oversight, so this pins the cost side of it: + a kind the analyzer adds must be a *visible* failure rather than a value silently coerced or + ignored, because every accessor keyed on ``kind`` would otherwise be quietly wrong. + + Which is also why the fix was to list the missing kind rather than to widen the field to + ``str``: widening would have turned this failure into silence for every future kind too. + """ + from cldk.models.go import GoAnalysis + from pydantic import ValidationError + + doc = json.loads(FOUR_KINDS.read_text(encoding="utf-8")) + module = next(iter(doc["application"]["symbol_table"].values())) + next(iter(module["types"].values()))["kind"] = "type_parameter" + with pytest.raises(ValidationError, match="kind"): + GoAnalysis(**doc) diff --git a/tests/analysis/go/test_go_dataflow.py b/tests/analysis/go/test_go_dataflow.py index ffafb8a1..0dd4ec30 100644 --- a/tests/analysis/go/test_go_dataflow.py +++ b/tests/analysis/go/test_go_dataflow.py @@ -73,30 +73,12 @@ def backend(httpsql): # ====================================================================================== -# The parse blocker (#436's first commit): GoType.kind is a Literal, so a kind the SDK -# does not list fails the WHOLE document rather than dropping a field. +# The L3/L4 overlays survive validation. The *parse blocker* that shipped alongside them +# (GoType.kind omitting "alias") is a model-level fact about -a 1 output with nothing to +# do with dataflow, so it is pinned in test_go_analysis.py beside the other model tests. # ====================================================================================== -def test_all_four_type_kinds_parse(): - """``type Alias = int`` made ``GoAnalysis(**doc)`` raise, which blocked ``-a 1`` output from any - project declaring an alias — nothing to do with dataflow. All four kinds must load.""" - analysis = GoAnalysis.model_validate_json(FOUR_KINDS.read_text(encoding="utf-8")) - kinds = {ty.kind for module in analysis.application.symbol_table.values() for ty in module.types.values()} - assert kinds == {"struct", "interface", "defined", "alias"} - - -def test_a_fifth_kind_is_refused_loudly_rather_than_dropped(): - """The ``Literal`` is deliberate, not an oversight: a kind the analyzer adds must be a visible - failure here and not a value silently coerced or ignored, because every accessor keyed on - ``kind`` would then be quietly wrong. ``extra="ignore"`` does not cover a ``Literal``.""" - doc = json.loads(FOUR_KINDS.read_text(encoding="utf-8")) - module = next(iter(doc["application"]["symbol_table"].values())) - next(iter(module["types"].values()))["kind"] = "type_parameter" - with pytest.raises(Exception, match="kind"): - GoAnalysis(**doc) - - def test_the_level_four_fixture_round_trips_losslessly(): """Every overlay the dataflow surface reads must survive validation — the regression that matters is the *silent* one, where a level-4 document parses and the graph is simply gone.""" From badc527bd41cdbf442f9215474fdcf71de0bf6bf Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Mon, 9 Nov 2026 12:50:01 -0500 Subject: [PATCH 11/15] test(go): sweep the dataflow relations over every position, not one The existing tests pin each property at one hand-picked position; these keep the relations and add the sampling. R1 depth monotonicity, R2 a capped slice is a prefix (not merely a subset) and reports the whole total, R3 flows_to_argument implies flows_to_call, R4 a returned path implies the predicate, R7 every edge endpoint is an id get_source can read -- each on both backends over every addressable position, capped at 60. This is the part of the analyzer's metamorphic design that transfers: it varies the input program and holds the analyzer fixed, which the SDK cannot do because it produces no analysis, so the SDK's dual is to vary the query and hold the analysis fixed. Not generative -- a property-based library over this space found nothing a deterministic sweep did not. The sweeps found a parity gap on their first run: get_source accepted a body-node id on the graph backend and raised on the local one, whose index held only types, callables and fields, though every edge endpoint and SliceNode.ref is one. Fixing it exposed describe diverging the other way -- local accepted a bare id, the graph silently dropped it. Both now accept both forms. Verified by mutation rather than by passing: six behavior-changing mutations in the shapes these relations guard are all caught. R7 anchors on GoBodyNode.id, the analyzer's own string, because comparing the SDK's composed id against the SDK's own lookup let a corrupted join move both sides together. Also tidies prose in the Go docstrings and tests. Refs #436 --- cldk/analysis/go/backend.py | 18 +- cldk/analysis/go/codeanalyzer/codeanalyzer.py | 47 ++- cldk/analysis/go/go_analysis.py | 2 +- cldk/analysis/go/neo4j/neo4j_backend.py | 33 +- docs/agent-api-reference.md | 15 + tests/analysis/go/test_go_dataflow.py | 4 +- tests/analysis/go/test_go_neo4j_live.py | 299 ++++++++++++++++++ 7 files changed, 400 insertions(+), 18 deletions(-) diff --git a/cldk/analysis/go/backend.py b/cldk/analysis/go/backend.py index a6423654..0d716100 100644 --- a/cldk/analysis/go/backend.py +++ b/cldk/analysis/go/backend.py @@ -169,7 +169,19 @@ def get_callees(self, callable_id: str) -> List[GoCallGraphEdge]: # -----[ source + whole-program ]----- @abstractmethod def get_source(self, node_id: str) -> str: - """The source text of the node ``node_id`` — ``module.source[node.span.bytes]``.""" + """The source text of the node ``node_id`` — ``module.source[node.span.bytes]``. + + ``node_id`` is a durable id (type, callable, field) **or a body-node id** — + ``"@"``, the analyzer's own, round-tripped and never composed + by the caller. The body-node form is what every edge endpoint and every + :class:`~cldk.analysis.commons.results.SliceNode` carries, so accepting it is what lets a + caller read an id off a ``get_ddg`` page or a slice and then read its text. + + Raises: + CodeanalyzerExecutionException: nothing in this application owns that id, or the node + exists and carries no span — a parameter-passing vertex is a dataflow position, + not a region of the file, and has no text on either backend. + """ @abstractmethod def get_call_graph_json(self) -> str: @@ -463,7 +475,7 @@ def get_degraded(self) -> List[GoDegraded]: **Read this before trusting an absence.** A non-empty list means some L4 step fell back to a coarser result (``range_func``, ``unmapped_ssa``, ``no_vta``, …), so a path that *is* returned is still a real may-flow, but "no path" is not a proof over the affected packages - and callables. :meth:`taint` reports this on its own ledger rather than making the judgement + and callables. :meth:`taint` reports this on its own ledger rather than making the judgment for the caller; this accessor is how a caller inspects it directly. """ @@ -509,7 +521,7 @@ def taint( ``unresolved``. **Sources, sinks and sanitizers are the caller's to supply.** This SDK ships no Go framework - catalogue and derives no default set: a per-language vocabulary of taint sources is policy + catalog and derives no default set: a per-language vocabulary of taint sources is policy that rots, and this accessor is the mechanism. **A sanitizer is two mechanisms wearing one word**, told apart by shape. A bare ``str`` cuts diff --git a/cldk/analysis/go/codeanalyzer/codeanalyzer.py b/cldk/analysis/go/codeanalyzer/codeanalyzer.py index 77a8f10e..79aec054 100644 --- a/cldk/analysis/go/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/go/codeanalyzer/codeanalyzer.py @@ -68,7 +68,7 @@ VIA, GoAnalysisBackend, ) -from cldk.analysis.go.ids import body_node_id, callable_match_names, external_signature, go_value_names, pick_declaration +from cldk.analysis.go.ids import body_node_id, callable_match_names, external_signature, go_value_names, pick_declaration, split_body_node_id from cldk.models.go import ( GoAnalysis, GoApplication, @@ -339,17 +339,27 @@ def get_functions(self) -> Dict[str, GoCallable]: return self._functions def get_source(self, node_id: str) -> str: - """The source text of ``node_id`` — ``module.source`` sliced by ``node.span.bytes`` — for a - durable-id node (type, callable, field). + """The source text of ``node_id`` — ``module.source`` sliced by ``node.span.bytes``. + + ``node_id`` is a durable-id node (type, callable, field) **or a body-node id**: a statement, + a call site or a vertex, named by the analyzer's own + ``"@"``. The body-node form is what every edge endpoint and + every :class:`~cldk.analysis.commons.results.SliceNode` carries, so without it a caller + could read an id off a ``get_ddg`` page and then not use it — and the Neo4j backend accepts + it (its ``:GoCanNode`` lookup does not care which kind of node it found), which made this + a parity gap rather than a shared limitation. Python's ``get_source`` names the same two + forms in its own contract. ``span.bytes`` are **UTF-8 byte** offsets, so a non-ASCII module is sliced on the encoded bytes and decoded back (an ASCII module is indexed directly, paying nothing), the same way the TS ``_Spanned.code`` property does. Slicing the ``str`` by character index would drift on any file with a multibyte character before the span. """ - entry = self._source_of.get(node_id) + entry = self._source_of.get(node_id) or self._body_source_of(node_id) if entry is None: - raise CodeanalyzerExecutionException(f"no source for {node_id!r}: not a type, callable or field of application " f"{self.application_name!r}") + raise CodeanalyzerExecutionException( + f"no source for {node_id!r}: not a type, callable, field or body node of application {self.application_name!r}" + ) mod, node = entry if node.span is None: raise CodeanalyzerExecutionException(f"node {node_id!r} carries no span, so it has no source text") @@ -362,6 +372,31 @@ def get_source(self, node_id: str) -> str: self._module_bytes[mod.id] = encoded return encoded[b0:b1].decode("utf-8") + def _body_source_of(self, node_id: str) -> "tuple | None": + """``(module, body node)`` for a body-node id, or ``None`` if nothing owns it. + + The owner is found with :func:`~cldk.analysis.go.ids.split_body_node_id` rather than by + splitting on ``@`` — trap 1: a library stand-in's own id holds ``/@external/`` before the + vertex ``@``. A stand-in has no module and so no text, which this reports as "not found" + rather than as a module lookup that would fail further on. + + Not indexed eagerly: the body-node ids outnumber the durable ones by a wide margin (3,324 + against 319 on the analyzer's own corpus), and ``get_source`` on one is the uncommon case, + so the prefix walk is paid per call instead of the index being paid per application. + """ + hit = split_body_node_id(node_id, self._dataflow_owners) + if hit is None: + return None + cid, key = hit + owner, _tid, path = self._dataflow_owners[cid] + if not path: # a stand-in: synthesized, so there is no source file behind it + return None + node = (owner.body or {}).get(key) or (owner.body or {}).get(key.lstrip("@")) + if node is None: + return None + entry = self._source_of.get(cid) + return (entry[0], node) if entry else None + # ===================================================================================== # L3/L4 DATAFLOW. See :class:`~cldk.analysis.go.backend.GoAnalysisBackend` for the contract and # for the four Go-specific traps; the id/name grammar those traps need is in @@ -392,7 +427,7 @@ def _require_dataflow(self) -> None: One guard for the per-callable graphs and the slices: they come from the same analyzer pass and go dark together. It **raises** rather than returning empty because at a shallower level - an empty answer would mean "not analysed" while looking exactly like "no dependence" (D7). + an empty answer would mean "not analyzed" while looking exactly like "no dependence" (D7). """ if self._level < self._DATAFLOW_LEVEL: raise CodeanalyzerUsageException( diff --git a/cldk/analysis/go/go_analysis.py b/cldk/analysis/go/go_analysis.py index 87144f5a..330ddd6c 100644 --- a/cldk/analysis/go/go_analysis.py +++ b/cldk/analysis/go/go_analysis.py @@ -257,7 +257,7 @@ def taint( m sources against n sinks in one traversal, reporting per pair: the witnesses in ``paths``, the pairs an absence claim can be built on in ``exhausted``, and everything else in ``unresolved``. The source/sink/sanitizer vocabulary is the caller's — no Go framework - catalogue ships. A non-empty :meth:`get_degraded` voids every ``exhausted`` certification, + catalog ships. A non-empty :meth:`get_degraded` voids every ``exhausted`` certification, because absence of a path is not a proof over an approximated graph. """ return self.backend.taint(sources, sinks, sanitizers, depth=depth, max_paths=max_paths) diff --git a/cldk/analysis/go/neo4j/neo4j_backend.py b/cldk/analysis/go/neo4j/neo4j_backend.py index afddfbc3..7978b8e5 100644 --- a/cldk/analysis/go/neo4j/neo4j_backend.py +++ b/cldk/analysis/go/neo4j/neo4j_backend.py @@ -102,7 +102,7 @@ def _slice_node(row: Dict[str, Any], module_key: Callable[[str], str]) -> SliceN The local twin is ``GoCodeanalyzer._slice_node``; the two share :func:`~cldk.analysis.commons.resolve.body_node_kind`, so a vertex a caller addressed through - ``resolve_value`` as a ``global`` comes back from a slice labelled a ``global`` on either + ``resolve_value`` as a ``global`` comes back from a slice labeled a ``global`` on either backend, and they differ only in where the fields are read from. ``row["var"]`` is the graph's name for the JSON's ``of`` (see @@ -480,7 +480,7 @@ def get_source(self, node_id: str) -> str: # the value a vertex stands for arrives under the name the shared helpers expect. # # A STAND-IN IS A FIRST-CLASS VERTEX. ``:GoExternal`` nodes carry ``GO_HAS_BODY_NODE`` body - # nodes at L4, so the traversals must not be labelled ``:GoCallable``-only on the interior or + # nodes at L4, so the traversals must not be labeled ``:GoCallable``-only on the interior or # a flow *through* ``database/sql.DB.Query`` would be invisible — which is the very flow the # httpsql fixture gates. ``:GoCallable|GoExternal`` is therefore deliberate everywhere below. # ===================================================================================== @@ -1051,20 +1051,41 @@ def get_degraded(self) -> List[GoDegraded]: def describe(self, nodes: Sequence[object]) -> List[SliceNode]: """Fill in ``source`` for these positions (see :meth:`GoAnalysisBackend.describe`). - One query for the batch rather than one per node. A vertex with no span gets ``None``: it is - a dataflow position, not a region of the file, and there is nothing to read on either - backend. + A vertex with no span gets ``None``: it is a dataflow position, not a region of the file, + and there is nothing to read on either backend. + + **A bare ``can://`` id is accepted as well as a resolved node**, because the local backend + accepts one and silently *dropping* it here was the worse half of that divergence: a caller + passing ids got a short list back with no signal, where the local backend answered. The + documented argument is a :class:`~cldk.analysis.commons.results.SliceNode`; an id is + resolved to one first rather than refused, so neither backend is the lenient outlier. """ out: List[SliceNode] = [] for n in nodes: if not isinstance(n, SliceNode): - continue + n = self._describe_id(str(n)) + if n is None: + continue try: out.append(n.model_copy(update={"source": self.get_source(n.ref)})) except CodeanalyzerExecutionException: out.append(n.model_copy(update={"source": None})) return out + def _describe_id(self, node_id: str) -> "SliceNode | None": + """One body-node id as a :class:`SliceNode`, or ``None`` if this graph holds no such node. + + The local twin is ``GoCodeanalyzer._slice_node``; both route the node's ``kind``/``var`` + through :func:`~cldk.analysis.commons.resolve.body_node_kind`, so an id described here and + one described in process carry the same caller-facing vocabulary. + """ + rows = self._run( + f"MATCH (b:GoBodyNode {{id:$id}}) OPTIONAL MATCH (c:{self._OWNER})-[:GO_HAS_BODY_NODE]->(b) " + "RETURN b.id AS ref, b.kind AS kind, b.var AS var, b.span_json AS span, c.signature AS callable, c.span_json AS c_span", + id=node_id, + ) + return _slice_node(self._span_lines(rows[0]), self._module_key) if rows else None + # -----[ artifact / dependency / config layer — reserved-but-unemitted on this projection ]----- def get_artifacts(self) -> Dict[str, PyArtifact]: raise CodeanalyzerExecutionException(ARTIFACT_LAYER_UNAVAILABLE.format(app=self.application_name)) diff --git a/docs/agent-api-reference.md b/docs/agent-api-reference.md index fbf22058..b11389e0 100644 --- a/docs/agent-api-reference.md +++ b/docs/agent-api-reference.md @@ -555,6 +555,21 @@ Five Go-specific facts, each one a thing that will mislead you if you don't know `exhausted` certification in a `taint` result and file an `analysis_degraded` diagnostic saying why. `complete` goes `False` with it. +**Reading text back off a position.** `get_source(node_id)` takes a durable id (type, callable, +field) **or a body-node id** — `"@"`, which is what every edge +endpoint and every `SliceNode.ref` carries. So an id read off a `get_ddg` page or a slice can be +turned into source directly: + +```python +page = go.get_ddg("sqlRepo.Find") +go.get_source(page.edges[0].src) # 'return s.db.Query("SELECT …")' +``` + +A parameter-passing vertex has no span — it is a dataflow position, not a region of the file — so +it raises rather than returning `""`, on both backends. `describe(nodes)` is the batch form: it +takes the `SliceNode`s a slice or path handed you, and also accepts bare ids, returning `source=None` +for a position that has no text rather than raising. + **The port-connectivity gate.** `slice_*`, `paths_between`, `flows_to_*` and `taint` refuse — naming the gap rather than returning a false empty — when no `formal_in` vertex in the attached analysis has an outgoing dependence edge. Same rule as Java's, and the probe reads the *data*, never a version diff --git a/tests/analysis/go/test_go_dataflow.py b/tests/analysis/go/test_go_dataflow.py index 0dd4ec30..b74191f6 100644 --- a/tests/analysis/go/test_go_dataflow.py +++ b/tests/analysis/go/test_go_dataflow.py @@ -5,7 +5,7 @@ gated analyzer-side by ``TestGateL4_HTTPToSQLPath``; ``four_type_kinds.json`` is one declaration of each ``GoType.kind`` at ``-a 1``. -**What is asserted and what is not.** These tests pin *behaviour a caller depends on* — that the +**What is asserted and what is not.** These tests pin *behavior a caller depends on* — that the gated flow is found, that the four Go id/name traps are handled, that a refutation is only made where one is warranted. They deliberately do not pin edge counts: those move with every analyzer release and a test that fails on an improvement teaches nothing. Where a count appears it is @@ -329,7 +329,7 @@ def test_paging_is_a_prefix_of_one_total_order(httpsql): def test_a_shallow_analysis_refuses_rather_than_answering_empty(tmp_path, monkeypatch): """At a level below the dataflow pass the three lists are absent, and an empty page there would - mean "not analysed" while looking exactly like "no dependence" (D7). So it raises, naming the + mean "not analyzed" while looking exactly like "no dependence" (D7). So it raises, naming the level to rebuild at.""" shallow = _facade(tmp_path, monkeypatch, FOUR_KINDS, app="kinds", level=AnalysisLevel.symbol_table) with pytest.raises(CodeanalyzerUsageException, match="level 3"): diff --git a/tests/analysis/go/test_go_neo4j_live.py b/tests/analysis/go/test_go_neo4j_live.py index 5a7f1c1c..e6a175e7 100644 --- a/tests/analysis/go/test_go_neo4j_live.py +++ b/tests/analysis/go/test_go_neo4j_live.py @@ -32,6 +32,7 @@ from cldk.analysis.commons.identity import resolve_identity from cldk.analysis.go.codeanalyzer import GoCodeanalyzer from cldk.analysis.go.neo4j import GoNeo4jBackend +from cldk.utils.exceptions.exceptions import CodeanalyzerExecutionException URI = os.environ.get("CLDK_TEST_GO_NEO4J_URI") USER = os.environ.get("CLDK_TEST_GO_NEO4J_USER", "neo4j") @@ -151,6 +152,70 @@ def _one_method_and_its_param(local): return None, None +#: Most positions a sweep will visit. The relation sweeps below are O(positions) or O(positions × +#: sinks), and the point of a sweep is to stop depending on *which* position was hand-picked — not +#: to visit every one of a 15,000-callable application. A cap keeps the suite usable on a large +#: corpus while still covering two orders of magnitude more ground than a single point. Positions +#: are taken in id order, so the sample is deterministic and a failure reproduces. +SWEEP_CAP = 60 + + +def _value_positions(local, *, cap=SWEEP_CAP, calling=None): + """``[(callable signature, formal_in name)]`` for plain parameters in the application. + + "Plain" excludes the ``:``/``:`` markers, not because those are uninteresting + but because they are already addressed by name elsewhere and would make the sweeps' failure + messages about the marker grammar rather than about the relation under test. + + **Enumerated from the local backend always, and swept on both.** The traversal needs the + symbol table, which only the in-process backend walks cheaply; the *positions* it yields are + plain ``(signature, name)`` strings that either backend resolves, so this is a property of the + application rather than of a backend. Enumerating per backend would also let the two sweep + different samples, which would quietly weaken every comparison below. + + ``calling``, when given, keeps only callables with a call edge to a signature in it. Without + that filter a cap takes the alphabetically-first positions, which on the analyzer's own corpus + are the ``asm``/``flow`` arithmetic fixtures — none of which call a library, so an + interprocedural relation samples 144 pairs that cannot possibly flow and never exercises + itself. Measured: the unfiltered sample yielded zero witnesses. + + Sorted, then capped — deterministic, so a violation reproduces on a re-run. + """ + wanted = None + if calling is not None: + targets = {t for t in calling} + wanted = {e.src for e in local.get_application_view().call_graph if e.dst in targets} + out = [] + for c, _tid, _path in local._iter_callables(): + sig = getattr(c, "signature", None) + if not sig or (wanted is not None and c.id not in wanted): + continue + for node in (c.body or {}).values(): + if node.kind == "formal_in" and node.of and not node.of.startswith("<"): + out.append((sig, node.of)) + return sorted(set(out))[:cap] + + +def _sink_arguments(local, *, cap=SWEEP_CAP): + """``([(stand-in signature, plain formal_in name)], {stand-in can:// id})``. + + Library stand-ins specifically: they are where real sinks live (``database/sql.DB.Query``), and + their ports are the ones an interprocedural relation has to cross a call boundary to reach. The + id set is returned alongside so :func:`_value_positions` can keep the callers that actually + reach them. + """ + out, ids = [], set() + for sym in local.get_external_symbols().values(): + if not sym.signature: + continue + sig = f"{sym.module}.{sym.signature}" + ports = [n.of for n in (sym.body or {}).values() if n.kind == "formal_in" and n.of and not n.of.startswith("<")] + if ports: + ids.add(sym.id) + out.extend((sig, of) for of in ports) + return sorted(set(out))[:cap], ids + + # -----[ standalone: the graph answers at all ]----- def test_neo4j_backend_populated(neo4j_backend): assert neo4j_backend.get_all_classes() or neo4j_backend.get_functions() @@ -377,6 +442,42 @@ def test_a_colliding_signature_fails_and_narrows_identically(dataflow_pair): assert lo_node.file == target +def test_source_and_describe_accept_a_body_node_id_on_both(dataflow_pair): + """``get_source`` and ``describe`` must take the ids the rest of the surface hands out. + + Every edge endpoint and every ``SliceNode.ref`` is a body-node id, so an accessor that cannot + read one leaves the caller holding an address it cannot use. Both diverged here and in opposite + directions, which is why this pins both: the local ``get_source`` indexed only types, callables + and fields and *raised* on a statement the graph read fine, while the graph's ``describe`` + silently *dropped* a bare id the local one accepted — a short list with no signal. + """ + local, graph = dataflow_pair + sig, _param = _one_method_and_its_param(local) + if sig is None: + pytest.skip("no callable with a ddg") + page = local.get_ddg(sig) + if not page.edges: + pytest.skip("that callable has no ddg edges") + # A statement endpoint -- one that has a span, so there is text to compare. + with_text = None + for e in page.edges: + for ref in (e.src, e.dst): + try: + if local.get_source(ref): + with_text = ref + break + except CodeanalyzerExecutionException: + continue + if with_text: + break + if with_text is None: + pytest.skip("no spanned endpoint on that callable's ddg") + assert local.get_source(with_text) == graph.get_source(with_text) + lo, ne = local.describe([with_text]), graph.describe([with_text]) + assert len(lo) == len(ne) == 1, "a bare id is accepted on one backend and dropped on the other" + assert (lo[0].ref, lo[0].kind, lo[0].source) == (ne[0].ref, ne[0].kind, ne[0].source) + + def test_the_three_graphs_agree_on_a_standin(dataflow_pair): """A stand-in has no statements, so all three graphs are empty — and both backends must *say* so rather than one crashing. The local backend read a field ``GoExternalSymbol`` does not have. @@ -410,3 +511,201 @@ def test_miss_paths_match(dataflow_pair): with pytest.raises(Exception) as ne: graph.resolve_value("definitely_not_a_value", within=sig) assert type(lo.value) is type(ne.value) + + +# ====================================================================================== +# RELATION SWEEPS (python-sdk#436 follow-up). +# +# The tests above pin each property at one hand-picked position. The relations are the +# same; what these add is the SAMPLING — each one sweeps every addressable position in +# the application instead of the one its author happened to choose. That is the only +# part of the analyzer's metamorphic design that transfers: the analyzer varies the +# INPUT PROGRAM and holds the analyzer fixed, which the SDK cannot do because it +# produces no analysis; so the SDK's dual is to vary the QUERY and hold the analysis +# fixed, and compare two of its own answers. +# +# Both backends are swept, not just the local one. These are single-backend invariants +# — nothing here compares local against graph — but an invariant that holds in process +# and not over Cypher is a bug in the Cypher, which is exactly the class of bug the +# parity tests above were written after finding four of. +# +# They are deliberately NOT generative. A property-based library over this space found +# nothing a sweep did not (measured on the 45-position httpsql fixture), so the cost of +# a new dependency buys sampling breadth that `_value_positions` already gives. +# +# A sweep reports the FIRST violating position, with the position in the message: a +# count tells you a relation broke, and the position tells you where to look. +# ====================================================================================== + + +@pytest.fixture(params=["local", "neo4j"]) +def swept(request, dataflow_pair): + """``(name, backend under test, backend to enumerate positions from)``. + + Each relation runs once per backend, named in the test id. Positions always come from the + *local* backend (see :func:`_value_positions`) so both sweeps visit the same sample. + """ + local, graph = dataflow_pair + return request.param, (local if request.param == "local" else graph), local + + +def test_a_forward_slice_is_monotone_in_depth(swept): + """R1. A deeper slice can only reach more, never less. + + The relation that would break if ``depth`` were applied after the walk rather than inside it, + or if the hop budget were counted per branch rather than per level — in which case a deeper + budget could *lose* a node by exhausting itself on another path first. + """ + name, b, src = swept + positions = _value_positions(src, cap=20) + if not positions: + pytest.skip("no plain value positions in this application") + checked = 0 + for sig, val in positions: + try: + near = {n.ref for n in b.slice_forward(val, within=sig, depth=1, max_nodes=100000).nodes} + far = {n.ref for n in b.slice_forward(val, within=sig, depth=None, max_nodes=100000).nodes} + except Exception: + continue # a position this backend cannot address is other tests' business + checked += 1 + assert near <= far, f"[{name}] {sig}/{val}: the depth=1 slice is not contained in the unbounded one" + assert checked, f"[{name}] every position raised; the sweep measured nothing" + + +def test_a_capped_slice_is_a_prefix_of_the_uncapped_one(swept): + """R2. ``max_nodes`` takes a *prefix* of one total order, not an arbitrary subset. + + Stronger than "is a subset", and the stronger form is the one a caller depends on: paging or + re-running with a larger cap must extend the previous answer rather than reshuffle it. A cap + applied before the sort — or a sort that is not total — satisfies "subset" and fails this. + """ + name, b, src = swept + positions = _value_positions(src, cap=20) + if not positions: + pytest.skip("no plain value positions in this application") + checked = 0 + for sig, val in positions: + try: + full = [n.ref for n in b.slice_forward(val, within=sig, depth=None, max_nodes=100000).nodes] + capped = b.slice_forward(val, within=sig, depth=None, max_nodes=2) + except Exception: + continue + checked += 1 + refs = [n.ref for n in capped.nodes] + assert refs == full[: len(refs)], f"[{name}] {sig}/{val}: the capped slice is not a prefix of the uncapped one" + assert capped.total == len(full), f"[{name}] {sig}/{val}: a capped slice misreports the whole size" + assert checked, f"[{name}] every position raised; the sweep measured nothing" + + +def test_flows_to_argument_implies_flows_to_call_everywhere(swept): + """R3. ``flows_to_argument`` is ``flows_to_call`` narrowed to one port, so the narrow answer + can never be ``True`` where the broad one is ``False``. + + A fact about their *targets* — one asks about a single ``formal_in``, the other about all of + them — so a violation means the two accessors disagree about what "entering this callable" + means, not that one of them has a bug in isolation. + """ + name, b, src = swept + sinks, sink_ids = _sink_arguments(src, cap=12) + positions = _value_positions(src, cap=12, calling=sink_ids) + if not positions or not sinks: + pytest.skip("this application has no value positions or calls nothing outside itself") + checked = 0 + for sig, val in positions: + for csig, arg in sinks: + try: + narrow = b.flows_to_argument(val, csig, arg, within=sig) + broad = b.flows_to_call(val, csig, within=sig) + except Exception: + continue + checked += 1 + assert not narrow or broad, f"[{name}] {sig}/{val} -> {csig}/{arg}: reaches the argument but not the call" + assert checked, f"[{name}] every triple raised; the sweep measured nothing" + + +def test_a_returned_path_implies_the_predicate(swept): + """R4. A witness implies the boolean: if ``paths_between`` returns a route from a value to a + callee's port, ``flows_to_argument`` must agree there is one. + + The direction matters. The converse is *not* asserted — a predicate may hold where no path is + returned, because the two need not agree on bounds — but a returned path that the predicate + denies would mean the SDK had produced a witness for something it says cannot happen, which is + the contradiction no caller could work around. + """ + name, b, src = swept + sinks, sink_ids = _sink_arguments(src, cap=12) + positions = _value_positions(src, cap=12, calling=sink_ids) + if not positions or not sinks: + pytest.skip("this application has no value positions or calls nothing outside itself") + checked, witnessed = 0, 0 + for sig, val in positions: + for csig, arg in sinks: + try: + paths = b.paths_between(val, arg, src_within=sig, dst_within=csig, depth=None).paths + pred = b.flows_to_argument(val, csig, arg, within=sig) + except Exception: + continue + checked += 1 + witnessed += bool(paths) + assert not paths or pred, f"[{name}] {sig}/{val} -> {csig}/{arg}: a path was returned but the predicate denies it" + assert checked, f"[{name}] every triple raised; the sweep measured nothing" + assert witnessed, f"[{name}] no pair in the sweep had a path, so the implication was never exercised" + + +def test_every_edge_endpoint_is_an_addressable_node(swept): + """R7. Every endpoint of every edge the three graphs return names a node the SDK can describe. + + The relation that closes the loop between the *page* accessors and the *position* accessors: an + endpoint the node index does not hold is an id a caller can read off an edge and then not use, + and ``describe``/``get_source`` would fail on it. This is the shape an id-joining or id-parsing + bug produces — both of which this branch had — at whichever callable the author did not pick. + """ + name, b, src = swept + sigs = sorted({sig for sig, _ in _value_positions(src, cap=25)}) + if not sigs: + pytest.skip("no addressable callables in this application") + # ``{signature: {the analyzer's own body-node ids}}`` — read off ``GoBodyNode.id``, which the + # emitter wrote and the SDK only mirrors, so it is an oracle the SDK's own id arithmetic + # cannot move. Callables whose nodes carry no ``id`` (below -a 3) contribute nothing. + truth = {} + for c, _tid, _path in src._iter_callables(): + ids = {n.id for n in (c.body or {}).values() if n.id} + if ids: + truth.setdefault(c.signature, set()).update(ids) + checked = 0 + for sig in sigs: + for getter in ("get_cfg", "get_cdg", "get_ddg"): + try: + page = getattr(b, getter)(sig) + except Exception: + continue + refs = sorted({ref for e in page.edges for ref in (e.src, e.dst)}) + if not refs: + continue + checked += len(refs) + # ANCHORED ON THE PAYLOAD'S OWN ``id``, not on whatever the SDK composed. The id that + # an edge endpoint carries and the id ``get_source`` looks up are built by the *same* + # join, so asking only "can get_source read this back" is self-consistent: corrupt the + # join and both sides move together, which is a mutation this sweep missed before the + # anchor was added. ``GoBodyNode.id`` is the analyzer's own string, independent of the + # SDK, so comparing against it is what makes a join bug visible. + owned = truth.get(sig, {}) + if owned: + unknown = [r for r in refs if r not in owned] + assert not unknown, ( + f"[{name}] {sig} {getter}: {len(unknown)} endpoint(s) are not ids the analyzer emitted " + f"for this callable, e.g. {unknown[0]} — the id join disagrees with the payload" + ) + # And the id must round-trip through the accessor whose contract names that form. + for ref in refs: + try: + text = b.get_source(ref) + except KeyError: + raise AssertionError(f"[{name}] {sig} {getter}: endpoint {ref} is an id the SDK cannot read source for") + except CodeanalyzerExecutionException as e: + # A vertex legitimately has no span and so no text; that is an answer, not a + # gap. An id the index does not hold at all is the failure this sweep is for. + assert "no span" in str(e), f"[{name}] {sig} {getter}: endpoint {ref} is not addressable — {e}" + continue + assert isinstance(text, str), f"[{name}] {sig} {getter}: {ref} returned {type(text).__name__}, not source text" + assert checked, f"[{name}] no edges in the sweep; the relation measured nothing" From 10617e77cd5a02e2c3f203940d29cf8121963c21 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 12 Nov 2026 10:51:07 -0500 Subject: [PATCH 12/15] fix(go): expand each node once in _reach, not once per predecessor Found by evaluating against the analyzer's own real-app corpus. _reach built each level as a comprehension and updated `seen` afterwards, so `seen` did not yet contain the level being built: a node with k predecessors in the current frontier was appended k times and the next level expanded every copy. The frontier tracked the number of paths rather than the number of nodes. The answer was never wrong -- a set cannot hold duplicates, and over 250 memos ports the reachable set is identical at every depth -- so only cost revealed it. On memos (208,293 source nodes, 743,154 edges): 151,417,783 frontier expansions at depth 8 against 670,450, 83.65s against 0.11s, and unbounded it did not terminate. depth=None is the default on paths_between, flows_to_call, flows_to_argument and taint, so the unusable case was the default one. After the fix, memos answers in 17s end to end: 11,477 ports swept at a max of 0.1s each, RSS flat at 3.4 GB, and taint finds 40 paths into database/sql.DB.QueryContext -- the HTTP-to-SQL-through-an-interface flow that app is in the corpus for. Pinned by counting expansions rather than by timing or frontier size: the committed fixture is too small to blow up, so the test wraps the adjacency and asserts each node is looked up at most once. It fails on the old walk. Refs #436 --- cldk/analysis/go/codeanalyzer/codeanalyzer.py | 25 +++++++- tests/analysis/go/test_go_dataflow.py | 57 +++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/cldk/analysis/go/codeanalyzer/codeanalyzer.py b/cldk/analysis/go/codeanalyzer/codeanalyzer.py index 79aec054..9cf354ee 100644 --- a/cldk/analysis/go/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/go/codeanalyzer/codeanalyzer.py @@ -730,12 +730,33 @@ def _reach(self, ref: str, direction: str, depth: int | None) -> set: Level-by-level rather than a plain stack, because ``depth`` is a hop budget and a depth-first walk cannot count hops without revisiting. Shared by the slices and by the two flow predicates, so "reachable" means one thing on this backend. + + **A node enters the next frontier once, which is the whole point of adding to ``seen`` + inside the loop rather than after it.** The obvious spelling builds the level as a + comprehension and updates ``seen`` afterwards:: + + nxt = [d for src in frontier for d in edges.get(src, ()) if d not in seen] + seen.update(nxt) + + That is correct but not usable: ``seen`` does not yet contain the level being built, so a + node with *k* predecessors in the current frontier is appended *k* times and the next level + expands every copy. The frontier then tracks the number of **paths** rather than the number + of nodes. Measured on memos (208,293 source nodes, 743,154 edges), over 250 ports: identical + answers at every depth — duplicates cannot change a set — but 151,417,783 frontier + expansions at depth 8 against 670,450, which is 83.65s against 0.11s, and unbounded it does + not terminate at all. Since ``depth=None`` is the default on + :meth:`paths_between`, :meth:`flows_to_call`, :meth:`flows_to_argument` and :meth:`taint`, + the unusable case was the default one. """ edges = self._sdg[0][direction] seen, frontier, hops = {ref}, [ref], 0 while frontier and (depth is None or hops < depth): - nxt = [d for src in frontier for d in edges.get(src, ()) if d not in seen] - seen.update(nxt) + nxt = [] + for src in frontier: + for d in edges.get(src, ()): + if d not in seen: + seen.add(d) + nxt.append(d) frontier = nxt hops += 1 return seen diff --git a/tests/analysis/go/test_go_dataflow.py b/tests/analysis/go/test_go_dataflow.py index b74191f6..e58ea3c5 100644 --- a/tests/analysis/go/test_go_dataflow.py +++ b/tests/analysis/go/test_go_dataflow.py @@ -16,6 +16,7 @@ """ import json +from collections import Counter from pathlib import Path from unittest.mock import MagicMock @@ -382,6 +383,62 @@ def test_max_paths_caps_witnesses_and_says_so(httpsql): assert len(capped.paths) <= 1 +def test_reach_expands_each_node_exactly_once(backend): + """``_reach`` must add to ``seen`` *inside* the level, not after it. + + The obvious spelling builds the level as a comprehension and updates ``seen`` afterwards, so + ``seen`` does not yet contain the level being built: a node with k predecessors in the current + frontier is appended k times, and the next level expands every copy. The frontier then tracks + the number of **paths** rather than the number of nodes. + + Only cost reveals it, because a set cannot hold duplicates and the answer stays correct — + measured on the memos corpus (208,293 source nodes, 743,154 edges), 151,417,783 frontier + expansions at depth 8 against 670,450, 83.65s against 0.11s, and unbounded it never terminated. + ``depth=None`` is the default on ``paths_between`` / ``flows_to_*`` / ``taint``, so the unusable + case was the default one. + + Pinned by counting expansions rather than by timing, which would be flaky, and rather than by + frontier size, which this fixture is too small to blow up: the adjacency is wrapped so every + lookup is recorded, and a correct walk looks each node up at most once. + """ + forward = backend._sdg[0]["forward"] + seeds = [r for r in backend._port_refs if forward.get(r)] + assert seeds, "fixture has no port with an outgoing edge" + + class Counting(dict): + def __init__(self, inner): + super().__init__(inner) + self.fetches = Counter() + + def get(self, key, default=None): + self.fetches[key] += 1 + return super().get(key, default) + + for ref in seeds: + counting = Counting(forward) + backend._sdg[0]["forward"] = counting + try: + backend._reach(ref, "forward", None) + finally: + backend._sdg[0]["forward"] = forward + repeats = {k: n for k, n in counting.fetches.items() if n > 1} + assert not repeats, f"{ref}: {len(repeats)} node(s) expanded more than once, e.g. {next(iter(repeats.items()))}" + + +def test_reach_is_monotone_and_terminates_unbounded(backend): + """An unbounded reach must terminate and contain every bounded one. + + The frontier bug made the unbounded case non-terminating on a real application while every + bounded case still answered, so a suite that only ever passed a `depth` would not have seen it. + """ + for ref in [r for r in backend._port_refs if backend._sdg[0]["forward"].get(r)][:12]: + unbounded = backend._reach(ref, "forward", None) + for depth in (1, 2, 3): + assert backend._reach(ref, "forward", depth) <= unbounded, f"{ref}: depth={depth} escapes the unbounded reach" + # And the walk is a fixed point: one more hop than the graph's diameter adds nothing. + assert backend._reach(ref, "forward", len(unbounded) + 1) == unbounded + + def test_the_two_slices_are_complementary(httpsql): """Forward is what a value affects, backward what affects it. The sink's port must be in the source's forward slice, and symmetrically.""" From 805a81b439d7defbeed85007bcac9f65c13cc780 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 12 Nov 2026 11:14:12 -0500 Subject: [PATCH 13/15] refactor: lift reach() into commons, fixing it for all four languages The frontier fix belonged in the language-independent code, not in Go's copy. _reach was byte-identical in the python, java, typescript and go local backends -- four copies of a graph algorithm over an adjacency of strings, which is the same argument that lifted shortest_walks in leg 2.5b. Four copies were four places to drift, and that is precisely how one bug came to live in all of them. commons.graphs.reach now owns the walk and the reason for its shape; each backend's _reach is a one-liner over it, keeping only `direction` -- which half of the cached adjacency to read -- as a backend concern. So the fix now covers Python, Java and TypeScript too, where the same unbounded walk was non-terminating on a sufficiently connected graph and depth=None is likewise the default on paths_between, flows_to_call, flows_to_argument and taint. Tests move to the shared suite beside call_reaches, the other lifted graph helper: a diamond chain where 55 nodes carry 262,144 paths pins expansion counting (1,048,573 frontier expansions before, 55 now), plus depth monotonicity, the fixed point, a cycle, an unknown seed, and that no backend keeps a copy. 2030 pass; the one failure predates the branch. Refs #436 --- cldk/analysis/commons/graphs.py | 47 ++++++++++ cldk/analysis/go/codeanalyzer/codeanalyzer.py | 40 ++------ .../java/codeanalyzer/codeanalyzer.py | 22 ++--- .../python/codeanalyzer/codeanalyzer.py | 23 +++-- .../typescript/codeanalyzer/codeanalyzer.py | 22 ++--- tests/analysis/commons/test_lifted_helpers.py | 91 +++++++++++++++++++ 6 files changed, 176 insertions(+), 69 deletions(-) diff --git a/cldk/analysis/commons/graphs.py b/cldk/analysis/commons/graphs.py index 7805e03d..72d4f603 100644 --- a/cldk/analysis/commons/graphs.py +++ b/cldk/analysis/commons/graphs.py @@ -422,6 +422,53 @@ def flow_path(nodes: Sequence[SliceNode], edges: Sequence[Tuple[str, "str | None return FlowPath(hops=[PathHop(frm=nodes[i], to=nodes[i + 1], via=via[rel], var=var, prov=list(prov or [])) for i, (rel, var, prov) in enumerate(edges)]) +def reach(edges: Mapping[str, Mapping[str, Sequence[tuple]]], src: str, depth: "int | None") -> set: + """The node ids reachable from ``src`` over ``edges`` in at most ``depth`` hops. + + ``edges`` is one direction of the ``{src: {dst: [label]}}`` adjacency each local backend + builds -- only the keys are read, so the labels are ignored here. ``depth`` of ``None`` is the + whole cone. + + Level-by-level rather than a plain stack, because ``depth`` is a hop budget and a depth-first + walk cannot count hops without revisiting. Every local backend's slices and flow predicates go + through this, so "reachable" means one thing per language *and* across them. + + **A node enters the next frontier once**, which is why ``seen`` is added to inside the level + rather than after it. The obvious spelling builds the level as a comprehension and updates + ``seen`` afterwards:: + + nxt = [d for src in frontier for d in edges.get(src, ()) if d not in seen] + seen.update(nxt) + + That is correct and unusable. ``seen`` does not yet contain the level being built, so a node + with *k* predecessors in the current frontier is appended *k* times and the next level expands + every copy: the frontier tracks the number of **paths**, not the number of nodes. A set cannot + hold duplicates, so the answer never changes -- which is exactly why it survived in four + separate copies of this function until an evaluation against a real application measured the + cost. On the ``memos`` corpus (208,293 source nodes, 743,154 edges), over 250 ports: identical + reachable sets at every depth, and 151,417,783 frontier expansions at depth 8 against 670,450 + -- 83.65s against 0.11s -- while unbounded it did not terminate at all. ``depth=None`` is the + default on ``paths_between`` / ``flows_to_call`` / ``flows_to_argument`` / ``taint``, so the + unusable case was the default one (python-sdk#436). + + Lifted out of the four language backends for :func:`shortest_walks`' reason, and with its + history as the argument: it is a graph algorithm over an adjacency of strings and knows no + language, and four copies were four places to drift -- which is how one bug came to live in + all of them. + """ + seen, frontier, hops = {src}, [src], 0 + while frontier and (depth is None or hops < depth): + nxt = [] + for node in frontier: + for d in edges.get(node, ()): + if d not in seen: + seen.add(d) + nxt.append(d) + frontier = nxt + hops += 1 + return seen + + def shortest_walks( edges: Mapping[str, Mapping[str, Sequence[tuple]]], src: str, diff --git a/cldk/analysis/go/codeanalyzer/codeanalyzer.py b/cldk/analysis/go/codeanalyzer/codeanalyzer.py index 9cf354ee..e9311b48 100644 --- a/cldk/analysis/go/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/go/codeanalyzer/codeanalyzer.py @@ -49,7 +49,7 @@ check_page_size, edge_page, ) -from cldk.analysis.commons.graphs import call_reaches, cone_sinks, flow_path, shortest_walks, slice_resolved, under_callable +from cldk.analysis.commons.graphs import call_reaches, cone_sinks, flow_path, reach, shortest_walks, slice_resolved, under_callable from cldk.analysis.commons.levels import analyzer_level from cldk.analysis.commons.resolve import ( CallableCandidate, @@ -727,39 +727,13 @@ def get_ddg(self, callable: str, *, in_class: str | None = None, page_size: int def _reach(self, ref: str, direction: str, depth: int | None) -> set: """The node ids reachable from ``ref`` in at most ``depth`` hops. - Level-by-level rather than a plain stack, because ``depth`` is a hop budget and a - depth-first walk cannot count hops without revisiting. Shared by the slices and by the two - flow predicates, so "reachable" means one thing on this backend. - - **A node enters the next frontier once, which is the whole point of adding to ``seen`` - inside the loop rather than after it.** The obvious spelling builds the level as a - comprehension and updates ``seen`` afterwards:: - - nxt = [d for src in frontier for d in edges.get(src, ()) if d not in seen] - seen.update(nxt) - - That is correct but not usable: ``seen`` does not yet contain the level being built, so a - node with *k* predecessors in the current frontier is appended *k* times and the next level - expands every copy. The frontier then tracks the number of **paths** rather than the number - of nodes. Measured on memos (208,293 source nodes, 743,154 edges), over 250 ports: identical - answers at every depth — duplicates cannot change a set — but 151,417,783 frontier - expansions at depth 8 against 670,450, which is 83.65s against 0.11s, and unbounded it does - not terminate at all. Since ``depth=None`` is the default on - :meth:`paths_between`, :meth:`flows_to_call`, :meth:`flows_to_argument` and :meth:`taint`, - the unusable case was the default one. + One line over :func:`~cldk.analysis.commons.graphs.reach`, which is where the walk and the + reason for its shape live: it is a graph algorithm over an adjacency of strings and knows no + language. This method stays because the *direction* is a backend concern — which half of the + cached adjacency to read — and because the slices and both flow predicates call it, so + "reachable" means one thing here. """ - edges = self._sdg[0][direction] - seen, frontier, hops = {ref}, [ref], 0 - while frontier and (depth is None or hops < depth): - nxt = [] - for src in frontier: - for d in edges.get(src, ()): - if d not in seen: - seen.add(d) - nxt.append(d) - frontier = nxt - hops += 1 - return seen + return reach(self._sdg[0][direction], ref, depth) def _slice_from(self, root: SliceNode, direction: str, depth: int | None, max_nodes: int) -> Slice: """:meth:`_reach`'s closure from ``root``, described and capped as the graph's is. diff --git a/cldk/analysis/java/codeanalyzer/codeanalyzer.py b/cldk/analysis/java/codeanalyzer/codeanalyzer.py index aaf1fd70..6e035e70 100644 --- a/cldk/analysis/java/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/java/codeanalyzer/codeanalyzer.py @@ -41,7 +41,7 @@ from cldk.analysis.commons.identity import resolve_identity, validate_identity, cache_matches from cldk.analysis import AnalysisLevel from cldk.analysis.commons.bounds import DEFAULT_PAGE_SIZE, check_page_size, edge_page -from cldk.analysis.commons.graphs import flow_path, shortest_walks, slice_resolved, under_callable +from cldk.analysis.commons.graphs import flow_path, reach, shortest_walks, slice_resolved, under_callable from cldk.analysis.commons.levels import ANALYZER_LEVELS, LEVEL_NAMES, analyzer_level from cldk.analysis.commons.results import Diagnostic, EdgePage, FlowPath, FlowPaths, Slice, SliceNode from cldk.analysis.java.backend import ( @@ -547,17 +547,15 @@ def link(src: str, dst: str, label: tuple) -> None: return self._sdg_cache def _reach(self, ref: str, direction: str, depth: int | None) -> set: - """The set of node ids reachable from ``ref`` in at most ``depth`` hops. Level by level - rather than a plain stack, because ``depth`` is a hop budget and a depth-first walk cannot - count hops without revisiting.""" - edges = self._sdg()[0][direction] - seen, frontier, hops = {ref}, [ref], 0 - while frontier and (depth is None or hops < depth): - nxt = [d for src in frontier for d in edges.get(src, ()) if d not in seen] - seen.update(nxt) - frontier = nxt - hops += 1 - return seen + """The set of node ids reachable from ``ref`` in at most ``depth`` hops. + + The walk is :func:`~cldk.analysis.commons.graphs.reach`, lifted there with its + measurements (python-sdk#436). The level-by-level shape and the reason a node must enter + the frontier exactly once live with it; the spelling that was here appended a node once per + predecessor, which left an unbounded walk non-terminating on a real application while every + answer it did return was correct. ``direction`` stays a backend concern. + """ + return reach(self._sdg()[0][direction], ref, depth) def _value_slice(self, root: SliceNode, *, backward: bool, depth: int | None, max_nodes: int) -> Slice: """See :meth:`JavaAnalysisBackend._value_slice`. The whole closure is computed and then cut, diff --git a/cldk/analysis/python/codeanalyzer/codeanalyzer.py b/cldk/analysis/python/codeanalyzer/codeanalyzer.py index f29a38b8..1cea90ab 100644 --- a/cldk/analysis/python/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/python/codeanalyzer/codeanalyzer.py @@ -64,7 +64,7 @@ from cldk.analysis.commons.keys import module_scope from cldk.analysis.commons.identity import resolve_identity, validate_identity from cldk.analysis import AnalysisLevel -from cldk.analysis.commons.graphs import call_reaches, under_callable +from cldk.analysis.commons.graphs import call_reaches, reach, under_callable from cldk.analysis.commons.levels import ANALYZER_LEVELS, LEVEL_NAMES, analyzer_level from cldk.analysis.commons.resolve import CallableCandidate, body_node_kind, resolve_callable_signature, resolve_value_name, resolve_within, value_candidate from cldk.analysis.commons.results import BodyRef, CallableRef, Diagnostic, EdgePage, EntrypointCoverage, FlowPath, FlowPaths, LocateResult, ModuleRef, Slice, SliceNode, TypeRef @@ -1333,18 +1333,17 @@ def link(src: str, dst: str, label: tuple) -> None: def _reach(self, ref: str, direction: str, depth: int | None) -> set: """The set of node ids reachable from ``ref`` in at most ``depth`` hops. - Level-by-level rather than a plain stack, because ``depth`` is a hop budget and a - depth-first walk cannot count hops without revisiting. Shared by the slices and by the two - mixed flow queries so "reachable" means one thing on this backend. + Shared by the slices and by the two mixed flow queries so "reachable" means one thing on + this backend. + + The walk is :func:`~cldk.analysis.commons.graphs.reach`, lifted there with its + measurements (python-sdk#436). The level-by-level shape and the reason a node must enter + the frontier exactly once live with it; the spelling that was here appended a node once per + predecessor, which left an unbounded walk non-terminating on a real application while every + answer it did return was correct. ``direction`` stays a backend concern -- which half of the + cached adjacency to read. """ - edges = self._sdg()[0][direction] - seen, frontier, hops = {ref}, [ref], 0 - while frontier and (depth is None or hops < depth): - nxt = [d for src in frontier for d in edges.get(src, ()) if d not in seen] - seen.update(nxt) - frontier = nxt - hops += 1 - return seen + return reach(self._sdg()[0][direction], ref, depth) def _slice_from(self, root: SliceNode, direction: str, depth: int | None, max_nodes: int) -> Slice: """:meth:`_reach`'s closure from ``root``, described and capped like the graph's. diff --git a/cldk/analysis/typescript/codeanalyzer/codeanalyzer.py b/cldk/analysis/typescript/codeanalyzer/codeanalyzer.py index dc623f6d..3cb79a24 100644 --- a/cldk/analysis/typescript/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/typescript/codeanalyzer/codeanalyzer.py @@ -53,7 +53,7 @@ check_page_size, edge_page, ) -from cldk.analysis.commons.graphs import call_reaches, cone_sinks, flow_path, shortest_walks, slice_resolved, under_callable +from cldk.analysis.commons.graphs import call_reaches, cone_sinks, flow_path, reach, shortest_walks, slice_resolved, under_callable from cldk.analysis.commons.keys import body_key_column, resolve_module_key from cldk.analysis.commons.levels import ANALYZER_LEVELS, LEVEL_NAMES, analyzer_level from cldk.analysis.commons.resolve import CallableCandidate, resolve_callable_signature, resolve_value_name, resolve_within @@ -1186,18 +1186,16 @@ def link(src: str, dst: str, label: tuple) -> None: def _reach(self, ref: str, direction: str, depth: int | None) -> set: """The set of node ids reachable from ``ref`` in at most ``depth`` hops. - Level-by-level rather than a plain stack, because ``depth`` is a hop budget and a - depth-first walk cannot count hops without revisiting. Shared by the slices and by the two - flow predicates so "reachable" means one thing on this backend. + Shared by the slices and by the two flow predicates so "reachable" means one thing on this + backend. + + The walk is :func:`~cldk.analysis.commons.graphs.reach`, lifted there with its + measurements (python-sdk#436). The level-by-level shape and the reason a node must enter + the frontier exactly once live with it; the spelling that was here appended a node once per + predecessor, which left an unbounded walk non-terminating on a real application while every + answer it did return was correct. ``direction`` stays a backend concern. """ - edges = self._sdg[0][direction] - seen, frontier, hops = {ref}, [ref], 0 - while frontier and (depth is None or hops < depth): - nxt = [d for src in frontier for d in edges.get(src, ()) if d not in seen] - seen.update(nxt) - frontier = nxt - hops += 1 - return seen + return reach(self._sdg[0][direction], ref, depth) def _slice_node(self, ref: str) -> SliceNode: """One reached body node in the caller's vocabulary. diff --git a/tests/analysis/commons/test_lifted_helpers.py b/tests/analysis/commons/test_lifted_helpers.py index d1421d2e..5d1c0c1c 100644 --- a/tests/analysis/commons/test_lifted_helpers.py +++ b/tests/analysis/commons/test_lifted_helpers.py @@ -486,3 +486,94 @@ def test_the_local_acceptance_domain_is_delimited_too(module, cls): src = inspect.getsource(getattr(importlib.import_module(module), cls)._edge_vars_in) assert "under_callable(" in src, "the acceptance domain stopped delimiting; a sibling callable's vars leak in" assert ".startswith(" not in src, "a bare prefix accepts a variable the delimited cut cannot sever" + + +# -----[ reach: the walk every local backend shares (python-sdk#436) ]----- +# +# Lifted out of four byte-identical copies, which is how one bug came to live in all of them: +# each built the level as a comprehension and updated ``seen`` afterwards, so a node with k +# predecessors in the current frontier was appended k times and the next level expanded every +# copy. The reachable SET never changed -- a set cannot hold duplicates -- so only cost showed it. + + +def _diamond_chain(levels): + """``n0 -> {a,b} -> n1 -> {a,b} -> n2 ...``: two predecessors per node, every level. + + The shape the bug is quadratic-or-worse on, and small enough to assert exactly: ``1 + 3*levels`` + nodes, but ``2**levels`` distinct paths from the head, so a frontier that counts paths rather + than nodes grows exponentially while the answer stays the same size. + """ + edges = {} + for lvl in range(levels): + a, b, nxt = f"a{lvl}", f"b{lvl}", f"n{lvl + 1}" + edges[f"n{lvl}"] = {a: [()], b: [()]} + edges[a] = {nxt: [()]} + edges[b] = {nxt: [()]} + return edges, 1 + 3 * levels + + +def test_reach_expands_each_node_at_most_once(): + """The invariant, pinned by counting lookups rather than by timing. + + 18 levels is 55 nodes and 262,144 paths; the old walk made 1,048,573 frontier expansions for + them where this makes 55. Counting is the non-flaky way to say that. + """ + from cldk.analysis.commons.graphs import reach + + edges, expected = _diamond_chain(18) + + class Counting(dict): + def __init__(self, inner): + super().__init__(inner) + self.fetches = {} + + def get(self, key, default=None): + self.fetches[key] = self.fetches.get(key, 0) + 1 + return super().get(key, default) + + counting = Counting(edges) + got = reach(counting, "n0", None) + assert len(got) == expected + repeats = {k: n for k, n in counting.fetches.items() if n > 1} + assert not repeats, f"reach re-expanded {len(repeats)} node(s), e.g. {next(iter(repeats.items()))}" + + +def test_reach_is_bounded_monotone_and_reaches_a_fixed_point(): + """``depth`` is a hop budget: each bound contains the shallower ones, and past the graph's + diameter the answer stops growing. Unbounded must equal the fixed point, not exceed it.""" + from cldk.analysis.commons.graphs import reach + + edges, expected = _diamond_chain(6) + sizes = [len(reach(edges, "n0", d)) for d in range(1, 14)] + assert sizes == sorted(sizes), f"reach is not monotone in depth: {sizes}" + assert sizes[-1] == expected == len(reach(edges, "n0", None)) + for d in range(1, 14): + assert reach(edges, "n0", d) <= reach(edges, "n0", None) + + +def test_reach_handles_a_cycle_and_an_unknown_seed(): + """A cycle must terminate (``seen`` is the guard, not the depth bound), and a seed with no + outgoing edges is its own reach rather than an error -- ``reaches(x, x)``'s zero-hop rule is + ``call_reaches``' business, not this walk's.""" + from cldk.analysis.commons.graphs import reach + + cyclic = {"a": {"b": [()]}, "b": {"c": [()]}, "c": {"a": [()]}} + assert reach(cyclic, "a", None) == {"a", "b", "c"} + assert reach({}, "lonely", None) == {"lonely"} + assert reach(cyclic, "a", 1) == {"a", "b"} + + +def test_every_local_backend_delegates_to_the_shared_walk(): + """No backend may keep its own copy: four copies were four places to drift, and they did.""" + import inspect + + from cldk.analysis.go.codeanalyzer import GoCodeanalyzer + from cldk.analysis.java.codeanalyzer import JCodeanalyzer + from cldk.analysis.python.codeanalyzer import PyCodeanalyzer + from cldk.analysis.typescript.codeanalyzer import TSCodeanalyzer + + for cls in (PyCodeanalyzer, JCodeanalyzer, TSCodeanalyzer, GoCodeanalyzer): + src = inspect.getsource(cls._reach) + code = [ln.strip() for ln in src.splitlines() if ln.strip()] + assert any(ln.startswith("return reach(") for ln in code), f"{cls.__name__}._reach does not delegate" + assert not any("while frontier" in ln for ln in code), f"{cls.__name__}._reach still carries its own walk" From 5b641aecf005c60cf221d78e294a601c7bc620fa Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 12 Nov 2026 15:31:51 -0500 Subject: [PATCH 14/15] perf(go): order the body-node owner candidates once, not per lookup split_body_node_id sorted every callable id on every call, and _body_source_of calls it once per get_source of a body-node id -- so a bulk get_source over a page's endpoints sorted 5,955 ids per endpoint on nats-server. The order is a property of the application, not of the node being looked up. longest_first() now produces it and the backend caches it in _owner_ids_longest_first. Ties break on the id, so the order is total and a lookup is reproducible rather than dependent on dict iteration. Found evaluating against the analyzer's real-app corpus. Refs #436 --- cldk/analysis/go/codeanalyzer/codeanalyzer.py | 22 ++++++++++++++---- cldk/analysis/go/ids.py | 23 ++++++++++++++++--- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/cldk/analysis/go/codeanalyzer/codeanalyzer.py b/cldk/analysis/go/codeanalyzer/codeanalyzer.py index e9311b48..b8b812c4 100644 --- a/cldk/analysis/go/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/go/codeanalyzer/codeanalyzer.py @@ -68,7 +68,7 @@ VIA, GoAnalysisBackend, ) -from cldk.analysis.go.ids import body_node_id, callable_match_names, external_signature, go_value_names, pick_declaration, split_body_node_id +from cldk.analysis.go.ids import body_node_id, callable_match_names, external_signature, go_value_names, longest_first, pick_declaration, split_body_node_id from cldk.models.go import ( GoAnalysis, GoApplication, @@ -372,6 +372,16 @@ def get_source(self, node_id: str) -> str: self._module_bytes[mod.id] = encoded return encoded[b0:b1].decode("utf-8") + @cached_property + def _owner_ids_longest_first(self) -> Tuple[str, ...]: + """Every owner id, longest first — the candidate order a body-node id is split against. + + Cached because the order is a property of the application, not of the node being looked up: + computing it inside the lookup sorted 5,955 ids on every ``get_source`` of a body-node id + on nats-server. + """ + return longest_first(self._dataflow_owners) + def _body_source_of(self, node_id: str) -> "tuple | None": """``(module, body node)`` for a body-node id, or ``None`` if nothing owns it. @@ -380,11 +390,15 @@ def _body_source_of(self, node_id: str) -> "tuple | None": vertex ``@``. A stand-in has no module and so no text, which this reports as "not found" rather than as a module lookup that would fail further on. - Not indexed eagerly: the body-node ids outnumber the durable ones by a wide margin (3,324 - against 319 on the analyzer's own corpus), and ``get_source`` on one is the uncommon case, + Not indexed eagerly: the body-node ids outnumber the durable ones by a wide margin (1.69 + **million** against 5,955 on nats-server), and ``get_source`` on one is the uncommon case, so the prefix walk is paid per call instead of the index being paid per application. + + The *candidate order*, though, is paid once — :attr:`_owner_ids_longest_first`. It used to + be sorted inside the lookup, which made a bulk ``get_source`` over a page's endpoints sort + every callable id per endpoint. """ - hit = split_body_node_id(node_id, self._dataflow_owners) + hit = split_body_node_id(node_id, self._owner_ids_longest_first) if hit is None: return None cid, key = hit diff --git a/cldk/analysis/go/ids.py b/cldk/analysis/go/ids.py index abd2eb20..183031b8 100644 --- a/cldk/analysis/go/ids.py +++ b/cldk/analysis/go/ids.py @@ -52,6 +52,20 @@ def body_node_id(callable_id: str, body_key: str) -> str: return f"{callable_id}{body_key}" if body_key.startswith("@") else f"{callable_id}@{body_key}" +def longest_first(callable_ids: Iterable[str]) -> Tuple[str, ...]: + """``callable_ids`` ordered longest first — the order :func:`split_body_node_id` needs. + + Longest first because a callable id can be a proper prefix of a nested closure's + (``…flow.Closure`` and ``…flow.Closure.closure_1``), so the shorter would otherwise claim the + longer's vertices. Ties are broken by the id itself, so the order is total and a lookup is + reproducible rather than dependent on dict iteration. + + Separate from the lookup so a backend computes it **once** per application instead of once per + node: see :func:`split_body_node_id`'s ``callable_ids``. + """ + return tuple(sorted(callable_ids, key=lambda cid: (-len(cid), cid))) + + def split_body_node_id(node_id: str, callable_ids: Iterable[str]) -> Tuple[str, str] | None: """``(callable id, body key)`` for ``node_id``, or ``None`` if no known callable owns it. @@ -72,15 +86,18 @@ def split_body_node_id(node_id: str, callable_ids: Iterable[str]) -> Tuple[str, Args: node_id: A body node's absolute ``can://`` id. - callable_ids: Every callable id in the application, stand-ins included. Order is - irrelevant; this sorts. + callable_ids: Every callable id in the application, stand-ins included, **already ordered + longest first** — :func:`longest_first` is what produces that order, and a backend + should compute it once rather than per call. Sorting here instead cost a sort of every + callable id on every lookup, which on nats-server (5,955 callables) made a bulk + ``get_source`` over a page's endpoints quadratic; the eval surfaced it. Returns: ``(callable_id, body_key)`` where ``body_node_id(*result) == node_id``, or ``None`` when no candidate owns it — a miss is returned, not raised, because the callers here are describing a walk's nodes and an unknown node is a node to skip, not an error to propagate. """ - for cid in sorted(callable_ids, key=len, reverse=True): + for cid in callable_ids: if not node_id.startswith(cid) or node_id == cid: continue rest = node_id[len(cid) :] From 6a1126fb0dd255d655999af3c6ca2bcecdc5dd56 Mon Sep 17 00:00:00 2001 From: Saurabh Sinha Date: Thu, 12 Nov 2026 15:37:14 -0500 Subject: [PATCH 15/15] docs: record the shared reach() walk in the language table The dataflow note covers Go's #436 surface; the reach lift is cross-language and belongs beside the other all-languages entries, with #446 named as the one limitation left on that surface. --- CLAUDE.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 89a7e609..466a3fea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,6 +45,19 @@ absence claim in the result; `max_paths` caps per pair, never per call. The sour vocabulary is the caller's — no framework catalogue ships. Java shares the `formal_in` dependence probe above, so `taint` refuses on the same analyses the four verbs do. +**All four languages, since #436:** the reachability walk every local backend's slices and flow +predicates run is **`commons.graphs.reach`**, lifted out of four byte-identical copies for the +reason that lifted `shortest_walks` in leg 2.5b — it is a graph algorithm over an adjacency of +strings and knows no language. Each backend's `_reach` is a one-liner over it, keeping only +`direction` as its own concern. The lift carried a fix: a node must enter the next frontier **once**, +and the spelling in all four copies appended it once per predecessor, so the frontier tracked path +count rather than node count — answers were always right (a set holds no duplicates), but an +unbounded walk did not terminate on a real application, and `depth=None` is the default on +`paths_between` / `flows_to_*` / `taint`. Measured on nats-server: 151,417,783 frontier expansions +at depth 8 against 670,450. **Known limitation, #446:** `shortest_walks` still cannot produce a +witness for a flow `flows_to_*` confirms on a large application — its replay prunes on BFS layer +boundaries but not toward the sink. + **Go, since #436 (the L3/L4 train):** the dataflow surface, with `PythonAnalysisBackend`'s names and signatures — `get_cfg`/`get_cdg`/`get_ddg`, `resolve_callable`/`resolve_value`, `slice_backward`/`slice_forward`/`backward_cone`, `reaches`/`paths_between`,